diff --git a/.dockerignore b/.dockerignore index c4997d6d..b7b551a8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,5 @@ dist *.md tests coverage -sdk-python -sdk-go +sdk .github diff --git a/.env.example b/.env.example index 7ac109c9..15393eb4 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,36 @@ -# LibScope Environment Variables -# Copy this file to .env and fill in your values. +# LibScope environment variables +# LibScope does not load .env files. Export these in your shell, or put them in +# the "env" block of your MCP client config. This file is a reference only. +# Every config key has a variable LIBSCOPE_
_; see +# docs/reference/configuration.md for the full list. # Embedding provider: "local" | "ollama" | "openai" (default: "local") LIBSCOPE_EMBEDDING_PROVIDER=local -# Ollama settings (required when provider is "ollama") -LIBSCOPE_OLLAMA_URL=http://localhost:11434 -LIBSCOPE_OLLAMA_MODEL=nomic-embed-text +# Embedding model and server URL (the local provider ignores both) +# LIBSCOPE_EMBEDDING_MODEL=nomic-embed-text +# LIBSCOPE_EMBEDDING_URL=http://localhost:11434 -# OpenAI settings (required when provider is "openai") +# OpenAI key (used for embeddings and for the LLM). +# Precedence: LIBSCOPE_OPENAI_API_KEY > OPENAI_API_KEY > ~/.libscope/secrets.json. LIBSCOPE_OPENAI_API_KEY= + +# LLM for `ask`: "auto" (default) | "openai" | "anthropic" | "ollama" | "passthrough" +# LIBSCOPE_LLM_PROVIDER=auto +# LIBSCOPE_LLM_MODEL= +# LIBSCOPE_LLM_URL= + +# Anthropic key. Precedence: LIBSCOPE_ANTHROPIC_API_KEY > ANTHROPIC_API_KEY > ~/.libscope/secrets.json. +# LIBSCOPE_ANTHROPIC_API_KEY= + +# Fetching from private networks and self-signed TLS certificates (default: false) +# LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=false +# LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS=false + +# MCP server: optional toolsets ("admin" adds sync, install-pack, list-packs, reindex-documents) +# LIBSCOPE_MCP_TOOLSETS=admin + +# Not config keys: active workspace, REST API key, key that encrypts stored webhook secrets +# LIBSCOPE_WORKSPACE=default +# LIBSCOPE_API_KEY= +# LIBSCOPE_SECRET_KEY= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7eef0349..fa5a339b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,8 @@ jobs: with: node-version: 22 cache: npm - - run: npm ci + # Lint, format and typecheck load no native addon. + - run: npm ci --ignore-scripts - run: npm run lint - run: npm run format:check - run: npm run typecheck @@ -36,7 +37,10 @@ jobs: with: node-version: ${{ matrix.node-version }} cache: npm - - run: npm ci + - run: npm ci --ignore-scripts + # Build only the native addons the tests load: better-sqlite3, and sharp + # (@xenova/transformers loads it in the neural retrieval-quality suite). + - run: npm rebuild better-sqlite3 sharp - run: npm run test:coverage - name: Upload coverage if: matrix.node-version == 22 @@ -65,10 +69,14 @@ jobs: with: node-version: 22 cache: npm - - run: npm ci + - run: npm ci --ignore-scripts + # docs:check opens an in-memory better-sqlite3 database. + - run: npm rebuild better-sqlite3 - run: npm run build - name: Verify dist output run: | test -f dist/mcp/server.js test -f dist/cli/index.js test -f dist/core/index.js + - name: Check generated docs (run `npm run docs:gen` to update) + run: npm run docs:check diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index af8c40f2..94969281 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -36,7 +36,8 @@ jobs: node-version: 22 cache: npm registry-url: "https://registry.npmjs.org" - - run: npm ci + # tsc and npm publish load no native addon. + - run: npm ci --ignore-scripts - run: npm run build - run: npm publish --provenance --access public env: diff --git a/CLAUDE.md b/CLAUDE.md index b89685c0..11dd52a0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # libscope — Agent Guidelines -AI-powered knowledge base with MCP integration. TypeScript, Node.js >=20, CommonJS (ES2022 target). +AI-powered knowledge base with MCP integration. TypeScript, Node.js >=20, ES modules (`"type": "module"`, NodeNext, ES2022 target). ## Build & Test @@ -13,9 +13,13 @@ npm run format # Auto-format npm test # Run all tests via vitest npm run test:coverage # Coverage thresholds: statements 75%, branches 74%, functions 75%, lines 75% npm run build # tsc — outputs to dist/ +npm run docs:gen # After build: rewrite the generated blocks in docs/reference/*.md +npm run docs:check # After build: exit 1 if a generated block is out of date (CI runs this) ``` -CI runs on Node 20 and 22. CI checks: lint, format:check, typecheck, test:coverage, build. Build verifies `dist/mcp/server.js`, `dist/cli/index.js`, `dist/core/index.js` exist. +CI runs on Node 20 and 22. CI checks: lint, format:check, typecheck, test:coverage, build, docs:check. Build verifies `dist/mcp/server.js`, `dist/cli/index.js`, `dist/core/index.js` exist. + +When you change a CLI command or option, an operation's input schema or summary, an MCP tool, a REST route or a config key, run `npm run build && npm run docs:gen` and commit the changed `docs/reference/*.md`. The generator is `scripts/gen-docs.mjs`; do not edit text between `` and `` by hand. **Before every push, run this exact sequence locally:** ```bash @@ -67,38 +71,66 @@ When delegating to subagents for mechanical fixes: ## Project Structure +Every feature is one **operation** in `src/core/operations/`. The CLI, MCP server, REST API and SDK are thin adapters over the operations: they parse input, call `runOperation(op, ctx, input)` (or `startOperationTask` for long-running operations), and format the result. Put logic in an operation or a core module, never in a surface. + ``` src/ -├── cli/ # Commander.js CLI -│ ├── index.ts # Main entry (#!/usr/bin/env node) -│ └── commands/ # Each file exports registerCommands(program) -├── mcp/ # MCP server -│ ├── server.ts # MCP entry point -│ └── tools/ # Each file exports registerTools(server, db, provider) -├── lite/ # libscope/lite — embeddable semantic search (no CLI/MCP/connectors) -│ ├── index.ts # Public entrypoint — exports LibScopeLite + types -│ ├── core.ts # LibScopeLite class (index, search, getContext, ask, rate) -│ ├── types.ts # LiteOptions, LiteDoc, LiteSearchResult, etc. -│ ├── normalize.ts # Raw input → markdown (dispatches to core/parsers/) -│ └── chunker-treesitter.ts # Optional tree-sitter code chunker (TS/JS/Python) -├── core/ # Business logic (documents, search, indexing, packs, topics, etc.) -│ └── parsers/ # File format parsers (markdown, pdf, docx, html, epub, pptx, csv, yaml, json) -├── api/ # REST API server (routes, middleware, openapi spec) -├── web/ # Web UI / dashboard -├── db/ # SQLite via better-sqlite3 (schema, migrations, connection) -├── providers/ # Embedding providers (local/xenova, ollama, openai) -├── registry/ # Git-backed pack registries (config, git, sync, publish, search, checksum) -├── connectors/ # Third-party syncs (notion, slack, confluence, onenote, obsidian) -├── config.ts # Config loading: env vars > project .libscope.json > user ~/.libscope/config.json > defaults -├── errors.ts # Error hierarchy -├── logger.ts # Pino logger -└── LibScope.ts # Main public API class +├── core/ +│ ├── operations/ # The operation layer: one file per group (documents, search, links, graph, tags, +│ │ # topics, searches, packs, registries, connectors, admin, analytics, webhooks, tasks) +│ │ ├── types.ts # defineOperation, runOperation, startOperationTask, createOperationContext, ListResult +│ │ ├── schemas.ts # Shared zod field schemas (documentId, topic, sourceType, limit, tags, ...) +│ │ └── index.ts # OPERATIONS (every operation, in surface order), getOperation(name) +│ ├── bootstrap.ts # One startup for every surface: config -> db path -> migrate -> provider -> vector table +│ ├── ingest.ts # add: content, file, directory, URL (+ spider) or repository +│ ├── tasks.ts # Background task registry (AbortSignal, progress, 1 h retention) +│ ├── overview.ts # Counts, topics, packs, index model, health +│ ├── document-view.ts # get-document view (paged content, tags, links, ratings) +│ ├── rag.ts # answer() for ask (LLM or passthrough) +│ ├── index.ts # Package root exports: LibScope class, option/result types, errors +│ ├── parsers/ # File format parsers (markdown, pdf, docx, html, epub, pptx, csv, yaml, json, text) +│ └── ... # Business logic used by operations (documents, search, indexing, packs, topics, ...) +├── cli/ +│ ├── index.ts # Program, global options (--json, --verbose, --log-level, --workspace), error handler; exports `program` +│ └── commands/ # One file per command group; each exports register(program) +├── mcp/ +│ ├── server.ts # createMcpServer(): tools generated from operations (11 core + admin toolset); runStdioServer() +│ ├── format.ts # Compact text output of each tool +│ ├── errors.ts # withErrorHandling(), errorResponse() +│ └── main.ts # stdio entry point (npm run serve) +├── api/ +│ ├── routes.ts # API_ROUTES: one route per operation with `http`, plus /openapi.json and /api/v1/health +│ ├── adapter.ts # HTTP input -> operation input; 202 + task for long-running operations; error -> status +│ ├── openapi.ts # buildOpenApiSpec(API_ROUTES) from the operation schemas +│ ├── middleware.ts +│ └── server.ts # startApiServer({ db, provider, config }, options) +├── web/ # Dashboard server (server.ts) and page (dashboard.ts: do not edit) +├── lite/ # libscope/lite: createLite() preset of LibScope, createCodeChunker(), TreeSitterChunker, normalizeRawInput() +├── db/ # SQLite (schema.ts migrations, connection.ts, index-meta.ts embedding model of the index) +├── providers/ # Embedding providers (local/xenova, ollama, openai) +├── registry/ # Git-backed pack registries (config, git, sync, publish, search, resolve, checksum) +├── connectors/ # notion, slack, confluence, onenote, obsidian, docs; registry.ts (types, saved connections, sync) +├── config-schema.ts # The zod config schema: keys, defaults, env var names, docs table (getConfigKeyTable) +├── config.ts # loadConfig(), config get/set/unset, secrets.json +├── errors.ts # Error hierarchy +├── logger.ts # Pino logger (stderr) +└── LibScope.ts # SDK class: add/search/ask/askStream/overview + namespaces bound to operations +scripts/gen-docs.mjs # Fills the generated blocks in docs/reference/*.md from dist/ tests/ -├── unit/ # Fast, mocked — test modules in isolation -├── integration/ # Real SQLite DB, full workflows -└── fixtures/ # test-db.ts, mock-provider.ts, helpers.ts +├── unit/ # Fast, mocked — test modules in isolation (operations: tests/unit/operations/) +├── integration/ # Real SQLite DB, full workflows +└── fixtures/ # test-db.ts, mock-provider.ts, helpers.ts ``` +### Adding or changing a feature + +1. Add or change the operation in `src/core/operations/.ts` with `defineOperation({ name, group, summary, input, annotations, http, handler })`. Every input field has `.describe()`; defaults live in the schema only. Add it to the group's exported list (it then appears in `OPERATIONS`). +2. REST: setting `http: { method, path }` creates the route and its OpenAPI entry. `annotations.longRunning` makes the route answer `202` with a task. +3. MCP: tools are registered in `src/mcp/server.ts` (`registerCoreTools` / `registerAdminTools`) with a formatter from `format.ts`. Not every operation is a tool: saved searches, webhooks, analytics and registries are CLI/REST/SDK only. +4. CLI: add or change the command in `src/cli/commands/.ts`; it calls the operation and prints the result. +5. SDK: add the operation to its namespace in `NAMESPACES` in `src/LibScope.ts`. +6. Run `npm run build && npm run docs:gen` and commit the updated references. List removed or renamed names in `docs/migration-v2.md`. + ## Error Handling All custom errors extend `LibScopeError` with a `code` property and optional `cause`: @@ -107,22 +139,23 @@ All custom errors extend `LibScopeError` with a `code` property and optional `ca LibScopeError ├── DatabaseError ├── EmbeddingError - ├── ValidationError - ├── FetchError + ├── ValidationError (REST 400) + ├── FetchError (REST 502) ├── ConfigError - ├── DocumentNotFoundError - ├── ChunkNotFoundError - └── TopicNotFoundError + └── NotFoundError (REST 404; code names the resource, e.g. LINK_NOT_FOUND) + ├── DocumentNotFoundError + ├── ChunkNotFoundError + └── TopicNotFoundError ``` -Always use the appropriate error subclass. MCP tool handlers must be wrapped with `withErrorHandling()` from `src/mcp/errors.ts`. +Always use the appropriate error subclass. MCP tool handlers are wrapped with `withErrorHandling()` from `src/mcp/errors.ts` (`registerOperationTool` does this). The CLI prints every error once in `src/cli/errors.ts` (`✗ message` and a hint). ## Database - **Engine:** better-sqlite3 + sqlite-vec for vector search -- **Schema version:** 17 (migrations in `src/db/schema.ts`) +- **Schema version:** 19 (migrations in `src/db/schema.ts`) - **Adding migrations:** Increment `SCHEMA_VERSION`, add entry to `MIGRATIONS` object with the new version number as key -- **Key tables:** documents, chunks, chunks_fts (FTS5), chunk_embeddings (vector), topics, ratings, schema_version +- **Key tables:** documents, chunks, chunks_fts (FTS5), chunk_embeddings (vector), index_meta (embedding model of the index), topics, tags, ratings, document_links, saved_searches, webhooks, packs, schema_version. `connector_configs` is unused. ## Testing Patterns @@ -159,7 +192,7 @@ packs// ``` - Registry names: `/^[a-zA-Z0-9_-]+$/`, 2-64 chars -- Git URLs: https://, ssh://, or git@host:path (no embedded credentials) +- Registry URLs: https://, ssh://, SCP-style SSH (user@host:path) or file:/// (no embedded credentials) - Path segment validation: reject `..`, `/`, `\`, null bytes, non-alphanumeric (except `._-`) ## Logging @@ -168,6 +201,6 @@ Use `getLogger()` from `src/logger.ts` (pino). Create child loggers with `create ## Configuration -`loadConfig()` merges with precedence: env vars > project `.libscope.json` > user `~/.libscope/config.json` > defaults. 30-second cache TTL. +`src/config-schema.ts` is the one zod schema for every config key: defaults, validation, `config get/set/unset`, env var names and the docs table. `loadConfig()` merges with precedence: env vars > project `.libscope.json` > user `~/.libscope/config.json` > defaults. API keys (`openai.apiKey`, `anthropic.apiKey`) are read only from env vars and `~/.libscope/secrets.json`. 30-second cache TTL. -Key env vars: `LIBSCOPE_EMBEDDING_PROVIDER` (local|ollama|openai), `LIBSCOPE_LLM_PROVIDER` (openai|ollama|anthropic|passthrough), `LIBSCOPE_OPENAI_API_KEY`, `LIBSCOPE_ANTHROPIC_API_KEY`, `LIBSCOPE_OLLAMA_URL`, `LIBSCOPE_OLLAMA_MODEL`. +Every key `section.field` has the env var `LIBSCOPE_
_`. Key env vars: `LIBSCOPE_EMBEDDING_PROVIDER` (local|ollama|openai), `LIBSCOPE_EMBEDDING_MODEL`, `LIBSCOPE_EMBEDDING_URL`, `LIBSCOPE_LLM_PROVIDER` (auto|openai|anthropic|ollama|passthrough; default auto), `LIBSCOPE_LLM_MODEL`, `LIBSCOPE_LLM_URL`, `LIBSCOPE_OPENAI_API_KEY` (or `OPENAI_API_KEY`), `LIBSCOPE_ANTHROPIC_API_KEY` (or `ANTHROPIC_API_KEY`), `LIBSCOPE_MCP_TOOLSETS` (admin|all). Not config keys: `LIBSCOPE_WORKSPACE`, `LIBSCOPE_API_KEY` (REST auth), `LIBSCOPE_SECRET_KEY` (webhook secrets), `LIBSCOPE_VERBOSE`, `LIBSCOPE_GIT_TIMEOUT_MS`. Full list: `docs/reference/configuration.md`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dc5d039b..003b1974 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,8 +12,8 @@ cd libscope # Install dependencies npm install -# Initialize the database -npm run build && node dist/cli/index.js init +# Build, then check the setup (creates the database) +npm run build && node dist/cli/index.js doctor --fix # Run in development mode (watch) npm run dev @@ -21,37 +21,55 @@ npm run dev ## Scripts -| Command | Description | -| ----------------------- | -------------------------------- | -| `npm run build` | Compile TypeScript to `dist/` | -| `npm run dev` | Watch mode compilation | -| `npm run lint` | Run ESLint | -| `npm run lint:fix` | Run ESLint with auto-fix | -| `npm run format` | Format code with Prettier | -| `npm run format:check` | Check formatting without changes | -| `npm run typecheck` | Type-check without emitting | -| `npm test` | Run tests | -| `npm run test:watch` | Run tests in watch mode | -| `npm run test:coverage` | Run tests with coverage report | +| Command | Description | +| ----------------------- | ----------------------------------------------------------------------- | +| `npm run build` | Compile TypeScript to `dist/` | +| `npm run dev` | Watch mode compilation | +| `npm run lint` | Run ESLint | +| `npm run lint:fix` | Run ESLint with auto-fix | +| `npm run format` | Format code with Prettier | +| `npm run format:check` | Check formatting without changes | +| `npm run typecheck` | Type-check without emitting | +| `npm test` | Run tests | +| `npm run test:watch` | Run tests in watch mode | +| `npm run test:coverage` | Run tests with coverage report | +| `npm run docs:gen` | Update the generated blocks in `docs/reference` (after `npm run build`) | +| `npm run docs:check` | Fail when a generated block is out of date (runs in CI) | +| `npm run docs:dev` | Docs site dev server (VitePress) | +| `npm run docs:build` | Build the docs site | ## Project Structure ``` src/ -├── core/ # Business logic (indexing, search, ratings, topics, documents) -├── db/ # SQLite schema, migrations, connection management +├── core/ +│ ├── operations/ # The operation layer: every command, tool, route and SDK method calls one +│ ├── parsers/ # File format parsers +│ └── ... # Business logic (ingest, indexing, search, rag, documents, tasks, ...) +├── cli/ # CLI: index.ts and one file per command group in commands/ +├── mcp/ # MCP server: tools generated from operations +├── api/ # REST API: routes generated from operations, OpenAPI +├── web/ # Web dashboard +├── lite/ # libscope/lite: createLite(), code chunker, normalizeRawInput() +├── db/ # SQLite schema, migrations, connection ├── providers/ # Embedding providers (local, Ollama, OpenAI) -├── mcp/ # MCP server and tool definitions -├── cli/ # CLI entry point and commands -├── config.ts # Configuration management +├── connectors/ # Notion, Slack, Confluence, Obsidian, OneNote, doc sites +├── registry/ # Git pack registries +├── config-schema.ts # Config schema (keys, defaults, env vars) +├── config.ts # Config loading +├── LibScope.ts # Node.js SDK class ├── logger.ts # Structured logging (pino) -└── errors.ts # Custom error hierarchy +└── errors.ts # Error hierarchy +scripts/ +└── gen-docs.mjs # Fills the generated blocks of docs/reference tests/ ├── unit/ # Fast isolated tests with mocked dependencies -├── integration/ # Tests with real SQLite DB +├── integration/ # Tests with a real SQLite database └── fixtures/ # Test helpers, mock providers, sample data ``` +See the [Architecture Guide](docs/guide/architecture.md) for the operation layer and how to add an operation. The CLI, MCP tool, configuration and REST references in `docs/reference` are partly generated from the code: after you change a command, tool, config key or route, run `npm run build && npm run docs:gen` and commit the result. + ## Making Changes 1. **Create a branch** from `main` @@ -59,7 +77,8 @@ tests/ 3. **Add tests** for new functionality 4. **Run the full check suite:** ```bash - npm run lint && npm run typecheck && npm test + npm run format:check && npm run lint && npm run typecheck && npm test + npm run build && npm run docs:check ``` 5. **Commit** with a descriptive message following [Conventional Commits](https://www.conventionalcommits.org/): - `feat: add bulk import from directory` @@ -93,4 +112,4 @@ Open an issue on GitHub with: ## License -By contributing, you agree that your contributions will be licensed under the MIT License. +By contributing, you agree that your contributions will be licensed under the project's license (Business Source License 1.1, see `LICENSE`). diff --git a/Dockerfile b/Dockerfile index a8242a76..ebb0dd66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,36 @@ +# Debian (glibc) base: the local embedding model (onnxruntime-node) does not run on Alpine (musl). + # Build stage -FROM node:22-alpine AS builder +FROM node:22-bookworm-slim AS builder WORKDIR /app COPY package*.json ./ -RUN npm ci +# tsc needs no native addon; the production stage builds them. +RUN npm ci --ignore-scripts COPY . . RUN npm run build # Production stage -FROM node:22-alpine AS production +FROM node:22-bookworm-slim AS production WORKDIR /app -RUN apk add --no-cache python3 make g++ && \ - addgroup -g 1001 libscope && \ - adduser -u 1001 -G libscope -s /bin/sh -D libscope +RUN groupadd -g 1001 libscope && \ + useradd -u 1001 -g libscope -s /bin/sh -M libscope && \ + mkdir /data && chown libscope:libscope /data COPY package*.json ./ -RUN npm ci --omit=dev --ignore-scripts && \ - npm rebuild better-sqlite3 && \ - apk del python3 make g++ +RUN apt-get update && \ + apt-get install -y --no-install-recommends g++ make python3 && \ + npm ci --omit=dev --ignore-scripts && \ + npm rebuild better-sqlite3 sharp && \ + npm cache clean --force && \ + apt-get purge -y g++ make python3 && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* COPY --from=builder /app/dist ./dist USER libscope -EXPOSE 3420 3421 -ENV LIBSCOPE_DATA_DIR=/data +# Config, secrets, connections and workspace databases live in $HOME/.libscope. +ENV HOME=/data VOLUME ["/data"] +EXPOSE 3378 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \ - CMD node -e "fetch('http://localhost:3420/api/v1/stats').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))" + CMD node -e "fetch('http://localhost:3378/api/v1/health').then(r => process.exit(r.status < 500 ? 0 : 1)).catch(() => process.exit(1))" ENTRYPOINT ["node", "dist/cli/index.js"] -CMD ["serve"] +CMD ["serve", "api", "--host", "0.0.0.0", "--port", "3378"] diff --git a/README.md b/README.md index 92d7c76b..6089f3cf 100644 --- a/README.md +++ b/README.md @@ -9,51 +9,37 @@ [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=RobertLD_libscope&metric=bugs)](https://sonarcloud.io/summary/new_code?id=RobertLD_libscope) [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=RobertLD_libscope&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=RobertLD_libscope) -LibScope is a local knowledge base that makes your documentation searchable by AI assistants. Point it at markdown files, URLs, or connect it to Obsidian/Notion/Confluence/Slack, and it chunks, embeds, and indexes everything into a local SQLite database. Your AI tools query it through [MCP](https://modelcontextprotocol.io/) (Model Context Protocol) or a REST API. +LibScope is a local knowledge base that makes your documentation searchable by AI assistants. Add markdown files, PDFs, web pages, repositories, or connect Obsidian, Notion, Confluence, Slack and OneNote. LibScope chunks, embeds and indexes everything into a local SQLite database. Your AI tools query it through [MCP](https://modelcontextprotocol.io/) (Model Context Protocol), the CLI, a REST API or the Node.js SDK. -Everything runs on your machine. No cloud services required for basic usage — just `npm install` and go. +Everything runs on your machine. The default embedding model runs locally, so no cloud service is needed for indexing and search. -## Getting Started +**Upgrading from 1.x?** LibScope 2.0 renames many commands, MCP tools, REST routes, config keys and SDK methods. See [Migrating to LibScope 2.0](docs/migration-v2.md). + +## Quick start + +### CLI ```bash npm install -g libscope -# Set up the database -libscope init +# Add files, directories, web pages or GitHub/GitLab repositories +libscope add ./docs --library my-lib +libscope add https://docs.example.com/guide --spider --max-pages 50 +libscope add https://github.com/org/repo -# Index some docs -libscope add ./docs/getting-started.md --library my-lib -libscope add https://docs.example.com/guide -libscope import ./docs/ --library my-lib --extensions .md,.mdx - -# Search +# Search, and ask questions (ask needs an LLM: set an OpenAI or Anthropic key, or use Ollama) libscope search "how to authenticate" +libscope ask "How do I configure OAuth2?" -# Start the MCP server so your AI assistant can query it -libscope serve +# Check the setup: config, database, embedding model, index and LLM +libscope doctor ``` -On first run with the default embedding provider, LibScope downloads the [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) model (~80 MB). Subsequent runs use the cached model. - -### Document Format Support - -LibScope supports **Markdown** (`.md`, `.mdx`) and **plain text** natively. Additional formats are available via optional dependencies: - -| Format | Extension | Optional Dependency | Node.js Requirement | -| ------------ | --------- | ------------------- | ------------------- | -| PDF | `.pdf` | `pdf-parse` (v2) | ≥ 20.16 or ≥ 22.3 | -| Word | `.docx` | `mammoth` | Any | -| EPUB | `.epub` | `epub2` | Any | -| PowerPoint | `.pptx` | `pizzip` | Any | -| CSV | `.csv` | Built-in | Any | - -The `pdf-parse`, `mammoth`, `epub2`, and `pizzip` packages are listed as `optionalDependencies` and install automatically when the Node.js version is compatible. Note: binary `.ppt` files are not supported — only `.pptx`. +The database is created on first use. The first command that embeds text downloads the local model [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) (about 80 MB, first run only). -## Using with AI Assistants +### MCP (Claude, Cursor, VS Code, ...) -LibScope exposes an MCP server over stdio. Point your MCP-compatible client at it: - -**Cursor** — add to `~/.cursor/mcp.json`: +Add LibScope to your MCP client config: ```json { @@ -66,523 +52,74 @@ LibScope exposes an MCP server over stdio. Point your MCP-compatible client at i } ``` -**Claude Code:** +For Claude Code: ```bash claude mcp add --scope user libscope -- npx -y libscope serve ``` -Once connected, your assistant can search docs, submit new documents, rate content quality, and ask RAG-powered questions against your knowledge base. - -
-Full list of MCP tools - -| Tool | What it does | -| ---------------------- | --------------------------------------------------------- | -| `search-docs` | Semantic search with topic/library/version/rating filters | -| `ask-question` | RAG question-answering with source citations | -| `get-document` | Retrieve a document by ID | -| `list-documents` | List docs with filters | -| `list-topics` | Browse the topic hierarchy | -| `submit-document` | Index new content (raw text or a URL to fetch) | -| `update-document` | Update a document's title, content, or metadata | -| `delete-document` | Remove a document | -| `rate-document` | Rate a doc 1–5 with optional feedback and corrections | -| `suggest-tags` | Auto-suggest tags based on content analysis | -| `save-search` | Save a named search query with filters | -| `list-saved-searches` | List all saved searches | -| `run-saved-search` | Execute a saved search by name or ID | -| `delete-saved-search` | Delete a saved search | -| `link-documents` | Create a cross-reference between two documents | -| `get-document-links` | List all incoming and outgoing links for a document | -| `delete-link` | Remove a cross-reference link | -| `reindex-documents` | Re-embed chunks (useful after switching providers) | -| `health-check` | DB status, doc/chunk counts | -| `sync-obsidian-vault` | Sync an Obsidian vault | -| `sync-onenote` | Sync OneNote notebooks via Microsoft Graph | -| `sync-notion` | Sync Notion pages and databases | -| `sync-confluence` | Sync Confluence spaces | -| `sync-slack` | Sync Slack channels and threads | -| `install-pack` | Install a knowledge pack | -| `list-packs` | List installed or registry packs | - -
- -## Connectors - -LibScope can pull documentation from several platforms. Each connector handles incremental syncing so re-runs only process what changed. - -```bash -# Obsidian — parses wikilinks, frontmatter, embeds, tags -libscope connect obsidian /path/to/vault -libscope connect obsidian /path/to/vault --sync # incremental re-sync - -# Notion -libscope connect notion --token secret_abc123 - -# Confluence -libscope connect confluence \ - --url https://acme.atlassian.net \ - --email user@acme.com \ - --token $CONFLUENCE_TOKEN - -# Slack — index channel messages and threads -libscope connect slack --token xoxb-... --channels general,engineering - -# OneNote — device code auth via Microsoft Graph -export ONENOTE_CLIENT_ID=your-client-id -libscope connect onenote - -# GitHub / GitLab repos -libscope add-repo https://github.com/org/repo --branch main --path docs/ - -# Remove a connector's data -libscope disconnect obsidian /path/to/vault -``` - -
-Connector options reference - -**Obsidian:** `--topic-mapping frontmatter`, `--exclude "templates/*" "daily/*"`, `--sync` - -**Notion:** `--exclude page-id-1 db-id-2`, `--sync` - -**Confluence:** `--spaces ENG,DEVOPS`, `--exclude-spaces ARCHIVE` - -**Slack:** `--thread-mode aggregate|separate`, `--sync` - -**OneNote:** `--notebook "Work Notes"`, `--sync` - -**GitHub/GitLab:** `--token`, `--branch`, `--path`, `--extensions .md,.mdx,.rst` - -
- -## Search and RAG - -```bash -# Semantic search -libscope search "authentication best practices" -libscope search "API rate limiting" --library my-lib --topic security --limit 10 +The server has 11 core tools (`search`, `ask`, `get-document`, `list-documents`, `overview`, `submit-document`, `update-document`, `delete-document`, `rate-document`, `link-documents`, `task`). Set `LIBSCOPE_MCP_TOOLSETS=admin` to add `sync`, `install-pack`, `list-packs` and `reindex-documents`. Under MCP, `ask` returns the retrieved context by default, and your assistant writes the answer. -# Ask questions (needs an LLM provider configured — see Configuration below) -libscope ask "How do I configure OAuth2?" --library my-lib - -# Interactive REPL for iterative searching -libscope repl -``` - -Search uses sqlite-vec for vector similarity when available, with FTS5 full-text search as a fallback. Results automatically get a 1.5× title boost when the document title matches query words. You can also pass `--diversity 0.5` (0–1) for MMR-based diversity reranking. - -### Programmatic SDK - -LibScope exports two embeddable APIs: - -**`LibScope`** — full SDK with all features (connectors, topics, packs, etc.): +### SDK (Node.js) ```ts import { LibScope } from "libscope"; -const scope = LibScope.create(); -await scope.index({ title: "My Doc", content: "..." }); -const results = await scope.search("query"); -scope.close(); -``` - -**`LibScopeLite`** — lightweight embeddable class for external applications. No CLI, no MCP server, no connectors. Designed for embedding semantic search directly into other tools (MCP servers, VS Code extensions, CI scripts): - -```ts -import { LibScopeLite } from "libscope/lite"; +const scope = LibScope.create(); // the same database as the CLI and MCP server -const lite = new LibScopeLite({ dbPath: ":memory:" }); +await scope.add({ + title: "Auth Guide", + content: "# Authentication\n\nUse OAuth2...", + library: "my-lib", +}); +await scope.add("./docs"); -// Index documents (or code files via tree-sitter chunking) -await lite.indexBatch(docs, { concurrency: 4 }); +const { items } = await scope.search("how to authenticate"); +for (const hit of items) console.log(hit.documentId, hit.title, hit.score); -// Hybrid vector + FTS5 search -const results = await lite.search("how to authenticate"); - -// Get RAG context for injection into an external LLM prompt -const context = await lite.getContext("How does auth work?"); - -lite.close(); -``` - -Tree-sitter powered code indexing splits TypeScript, JavaScript, and Python files at function/class boundaries: - -```ts -import { TreeSitterChunker } from "libscope/lite"; - -const chunker = new TreeSitterChunker(); -const chunks = await chunker.chunk(sourceCode, "typescript"); -// Each chunk is a complete function or class with 1-based line numbers -``` - -See the [LibScope Lite guide](https://libscope.com/guide/lite) and [API reference](https://libscope.com/reference/lite-api) for the full documentation. - -## Organizing Content - -**Topics** give your docs a hierarchy. **Tags** give them flexible labels. **Workspaces** give you isolated databases. - -```bash -# Topics -libscope topics create "backend" -libscope topics create "auth" --parent backend --description "Auth & identity" - -# Tags -libscope tag add typescript,api,v2 - -# Workspaces — separate knowledge bases entirely -libscope workspace create my-project -libscope workspace use my-project -libscope --workspace my-project search "deploy steps" -``` +const result = await scope.ask("How do I configure OAuth2?"); // needs an LLM, like the CLI +if (result.mode === "answer") console.log(result.answer); -Documents also keep version history, so you can roll back if a re-index goes wrong: - -```bash -libscope docs history -libscope docs rollback 3 -``` - -## REST API - -For programmatic access outside of MCP: - -```bash -libscope serve --api --port 3378 -``` - -OpenAPI 3.0 spec at `GET /openapi.json`. Key endpoints: - -| Method | Endpoint | Description | -| --------------- | --------------------------------- | ---------------------------------- | -| `GET` | `/api/v1/search?q=...` | Semantic search | -| `POST` | `/api/v1/batch-search` | Batch search (up to 20 queries) | -| `POST` | `/api/v1/ask` | RAG question-answering | -| `GET/POST` | `/api/v1/documents` | List or create documents | -| `GET/PATCH/DELETE` | `/api/v1/documents/:id` | Get, update, or delete a document | -| `POST` | `/api/v1/documents/url` | Index from a URL | -| `POST` | `/api/v1/documents/:id/tags` | Add tags | -| `GET` | `/api/v1/documents/:id/suggest-tags` | Auto-suggest tags | -| `GET/POST` | `/api/v1/documents/:id/links` | List or create cross-references | -| `DELETE` | `/api/v1/links/:id` | Delete a cross-reference | -| `GET/POST` | `/api/v1/topics` | List or create topics | -| `GET` | `/api/v1/tags` | List tags | -| `GET/POST` | `/api/v1/searches` | List or create saved searches | -| `POST` | `/api/v1/searches/:id/run` | Run a saved search | -| `DELETE` | `/api/v1/searches/:id` | Delete a saved search | -| `POST` | `/api/v1/bulk/delete` | Bulk delete documents | -| `POST` | `/api/v1/bulk/retag` | Bulk add/remove tags | -| `POST` | `/api/v1/bulk/move` | Bulk move to a topic | -| `GET/POST` | `/api/v1/webhooks` | List or create webhooks | -| `DELETE` | `/api/v1/webhooks/:id` | Delete a webhook | -| `POST` | `/api/v1/webhooks/:id/test` | Send a test ping to a webhook | -| `GET` | `/api/v1/analytics/searches` | Search analytics and knowledge gaps| -| `GET` | `/api/v1/connectors/status` | Connector sync status and history | -| `GET` | `/api/v1/stats` | Usage statistics | -| `GET` | `/api/v1/health` | Health check | - -## Configuration - -LibScope reads config from (highest priority first): **environment variables** → **`.libscope.json`** in your project → **`~/.libscope/config.json`** → built-in defaults. - -### Embedding Providers - -| Provider | Default? | Requirements | Model | -| -------- | -------- | ----------------------------------------- | ---------------------- | -| `local` | Yes | None (~80 MB model download on first run) | all-MiniLM-L6-v2 | -| `ollama` | | Ollama running locally | nomic-embed-text | -| `openai` | | `LIBSCOPE_OPENAI_API_KEY` | text-embedding-3-small | - -```bash -libscope config set embedding.provider ollama -# or -export LIBSCOPE_EMBEDDING_PROVIDER=openai -export LIBSCOPE_OPENAI_API_KEY=sk-... -``` - -### LLM for RAG - -The `ask` command and `ask-question` MCP tool need an LLM. Configure one with: - -```bash -export LIBSCOPE_LLM_PROVIDER=openai # or ollama, anthropic -export LIBSCOPE_LLM_MODEL=gpt-4o-mini # optional -``` - -
-All environment variables - -| Variable | Description | Default | -| ---------------------------------- | ---------------------------------------- | ------------------------ | -| `LIBSCOPE_EMBEDDING_PROVIDER` | `local`, `ollama`, or `openai` | `local` | -| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key | — | -| `LIBSCOPE_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` | -| `LIBSCOPE_LLM_PROVIDER` | LLM for RAG (`openai` / `ollama` / `anthropic`) | — | -| `LIBSCOPE_LLM_MODEL` | LLM model override | — | -| `LIBSCOPE_ANTHROPIC_API_KEY` | Anthropic API key (for Claude models) | — | -| `LIBSCOPE_ALLOW_PRIVATE_URLS` | Allow fetching from private/internal IPs | `false` | -| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed TLS certificates | `false` | -| `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — | -| `ONENOTE_TENANT_ID` | Microsoft tenant ID | `common` | -| `NOTION_TOKEN` | Notion integration token | — | -| `CONFLUENCE_URL` | Confluence base URL | — | -| `CONFLUENCE_EMAIL` | Confluence user email | — | -| `CONFLUENCE_TOKEN` | Confluence API token | — | - -
- -
-Example config file (~/.libscope/config.json) - -```json -{ - "embedding": { - "provider": "local", - "ollamaUrl": "http://localhost:11434", - "ollamaModel": "nomic-embed-text", - "openaiModel": "text-embedding-3-small" - }, - "llm": { - "provider": "openai", - "model": "gpt-4o-mini", - "anthropicApiKey": "sk-ant-..." - }, - "database": { - "path": "~/.libscope/libscope.db" - }, - "logging": { - "level": "info" - }, - "indexing": { - "allowPrivateUrls": false, - "allowSelfSignedCerts": false - } -} -``` - -
- -### Corporate / Internal Networks - -If you're indexing docs from internal servers (Confluence, wikis, etc.), you may need: - -```bash -# Allow fetching from private/internal IP addresses -libscope config set indexing.allowPrivateUrls true - -# Accept self-signed or corporate TLS certificates -libscope config set indexing.allowSelfSignedCerts true -``` - -Or via environment variables: - -```bash -export LIBSCOPE_ALLOW_PRIVATE_URLS=true -export LIBSCOPE_ALLOW_SELF_SIGNED_CERTS=true -``` - -## Webhooks - -LibScope can push events to any HTTP endpoint. Useful for triggering CI pipelines, Slack notifications, or custom workflows whenever documents are created or updated. - -```bash -libscope serve --api # webhooks require the REST API -``` - -```bash -# Create a webhook -curl -X POST http://localhost:3378/api/v1/webhooks \ - -H "Content-Type: application/json" \ - -d '{"url": "https://hooks.example.com/libscope", "events": ["document.created", "document.updated"], "secret": "my-hmac-secret"}' - -# Send a test ping -curl -X POST http://localhost:3378/api/v1/webhooks//test +scope.close(); ``` -Webhook payloads are signed with HMAC-SHA256 when a secret is set. The signature is in the `X-LibScope-Signature` header. +To embed LibScope in another application without reading config files, use `createLite()` from `libscope/lite`. -Supported events: `document.created`, `document.updated`, `document.deleted`. +## Documentation -## Other Tools +The full documentation is at [libscope.com](https://libscope.com). -LibScope ships with a few more utilities beyond the core index-and-search loop: +Guides: -```bash -# Watch a directory and auto-reindex on changes -libscope watch ./docs/ +- [Getting started](https://libscope.com/guide/getting-started) +- [Configuration](https://libscope.com/guide/configuration): embedding providers, LLM for `ask`, API keys +- [MCP setup](https://libscope.com/guide/mcp-setup) +- [Connectors](https://libscope.com/guide/connectors): Obsidian, Notion, Confluence, Slack, OneNote, documentation sites +- [Knowledge packs](https://libscope.com/guide/knowledge-packs) and [pack registries](https://libscope.com/guide/pack-registries) +- [Programmatic usage](https://libscope.com/guide/programmatic-usage) and [LibScope Lite](https://libscope.com/guide/lite) +- [Web dashboard](https://libscope.com/guide/dashboard), [webhooks](https://libscope.com/guide/webhooks), [troubleshooting](https://libscope.com/guide/troubleshooting) -# Re-embed everything after switching embedding providers -libscope reindex +Reference (generated from the code): -# Find duplicate documents -libscope dedupe +- [CLI commands](docs/reference/cli.md) +- [MCP tools](docs/reference/mcp-tools.md) +- [REST API](docs/reference/rest-api.md) (`libscope serve api`) +- [Configuration keys and environment variables](docs/reference/configuration.md) -# Export / import the whole knowledge base -libscope export ./backup.json -libscope import-backup ./backup.json +Client SDKs for the REST API: [Python](sdk/python/README.md) and [Go](sdk/go/README.md). -# Usage analytics -libscope stats # overview -libscope stats popular # most-searched docs -libscope stats stale --days 90 # docs nobody searches for +## Supported formats -# Knowledge packs — portable document bundles -libscope pack create --name "react-docs" --topic react -libscope pack install ./react-docs.json -``` - -### Pack Registries +Markdown (`.md`, `.mdx`, `.markdown`), plain text, HTML, JSON, YAML, CSV, EPUB and PowerPoint (`.pptx`) are built in. PDF (`pdf-parse`) and Word `.docx` (`mammoth`) need optional dependencies, which npm installs when your Node.js version supports them. -Share and discover knowledge packs through git-based registries. A registry is a git repo with a defined folder structure managed by libscope. +## How it works -```bash -# Add a registry -libscope registry add https://github.com/org/libscope-registry.git --name official - -# Search for packs across all registries -libscope registry search "react" - -# Install a pack by name (resolves from registries) -libscope pack install react-docs -libscope pack install react-docs@1.2.0 # specific version - -# Create your own registry -libscope registry create ./my-registry - -# Publish a pack file to your registry -libscope registry publish ./my-pack.json -r my-registry --version 1.0.0 - -# Submit a pack to someone else's registry (creates a feature branch) -libscope registry publish ./my-pack.json -r community --submit -``` +LibScope stores everything in one SQLite database per workspace (`~/.libscope/workspaces/default/libscope.db` for the default workspace). The CLI, MCP server, REST API and SDK all open the same file and call the same operations, so they take the same parameters and return the same results. -Authentication is delegated to git — SSH keys and HTTPS credential helpers work automatically. Registries cache locally and support offline index lookups. See the [Pack Registries guide](/guide/pack-registries) for full details. - -There's also a web dashboard at `http://localhost:3377` when you run `libscope serve`, with search, document browsing, topic navigation, and a knowledge graph visualization at `/graph`. - -
-Full CLI reference - -**Core** - -| Command | Description | -| ----------------------------------- | ----------------------------------- | -| `libscope init` | Initialize the database | -| `libscope add ` | Index a file or URL | -| `libscope import ` | Bulk import from a directory | -| `libscope import-batch ` | Parallel batch import | -| `libscope search ` | Search | -| `libscope ask ` | RAG question-answering | -| `libscope repl` | Interactive search REPL | -| `libscope serve` | Start MCP server (`--api` for REST) | - -**Documents** - -| Command | Description | -| --------------------------------------- | ---------------------------- | -| `libscope docs list` | List documents | -| `libscope docs show ` | Show a document | -| `libscope docs update ` | Update title/content/metadata| -| `libscope docs delete ` | Delete a document | -| `libscope docs history ` | Version history | -| `libscope docs rollback ` | Roll back to a prior version | - -**Organization** - -| Command | Description | -| ------------------------------------ | -------------------------------- | -| `libscope topics list` | List topics | -| `libscope topics create ` | Create a topic | -| `libscope tag add ` | Add tags | -| `libscope tag remove ` | Remove a tag | -| `libscope tag list` | List tags | -| `libscope workspace create ` | Create workspace | -| `libscope workspace list` | List workspaces | -| `libscope workspace use ` | Switch workspace | -| `libscope workspace delete ` | Delete workspace | - -**Saved Searches** - -| Command | Description | -| ------------------------------------ | ----------------------------- | -| `libscope searches list` | List all saved searches | -| `libscope searches run ` | Re-run a saved search | -| `libscope searches delete ` | Delete a saved search | -| `libscope search --save ` | Save a search while running it| - -**Document Links** - -| Command | Description | -| ------------------------------------- | -------------------------------- | -| `libscope link ` | Create a cross-reference | -| `libscope links ` | Show all links for a document | -| `libscope unlink ` | Remove a link | -| `libscope prereqs ` | Show prerequisite reading chain | - -**Bulk Operations** - -| Command | Description | -| ---------------------------- | ----------------------------------- | -| `libscope bulk delete` | Delete all matching documents | -| `libscope bulk retag` | Add/remove tags on matching docs | -| `libscope bulk move` | Move matching docs to a topic | - -**Connectors** - -| Command | Description | -| ---------------------------------- | -------------------------- | -| `libscope connect obsidian ` | Sync Obsidian vault | -| `libscope connect onenote` | Sync OneNote | -| `libscope connect notion` | Sync Notion | -| `libscope connect confluence` | Sync Confluence | -| `libscope connect slack` | Sync Slack | -| `libscope add-repo ` | Index a GitHub/GitLab repo | -| `libscope disconnect ` | Remove connector data | - -**Registries** - -| Command | Description | -| ----------------------------------------------------- | ---------------------------------------- | -| `libscope registry add [-n ]` | Register a git repo as a pack registry | -| `libscope registry remove ` | Unregister a registry | -| `libscope registry list` | List configured registries | -| `libscope registry sync []` | Sync one or all registries | -| `libscope registry search [-r ]` | Search registry pack indexes | -| `libscope registry create ` | Initialize a new registry repo | -| `libscope registry publish -r ` | Publish a pack file to a registry | -| `libscope registry unpublish -r ` | Remove a pack version from a registry | - -**Utilities** - -| Command | Description | -| --------------------------------- | ---------------------------- | -| `libscope watch ` | Auto-reindex on file changes | -| `libscope reindex` | Re-embed all chunks | -| `libscope dedupe` | Find duplicates | -| `libscope export ` | Export to JSON | -| `libscope import-backup ` | Import from backup | -| `libscope stats` | Usage overview | -| `libscope pack install ` | Install a knowledge pack | -| `libscope pack create` | Create a knowledge pack | -| `libscope config set ` | Set config | -| `libscope config show` | Show config | - -**Global flags:** `--verbose`, `--log-level `, `--workspace ` - -
- -## How It Works - -LibScope stores everything in a local SQLite database (at `~/.libscope/libscope.db` by default): - -- Documents are split into chunks by heading boundaries -- Each chunk is embedded into a vector using the configured provider -- Vector search is done via [sqlite-vec](https://github.com/asg017/sqlite-vec); FTS5 full-text search is used as a fallback -- The MCP server reads from this same database over stdio -- Connectors fetch content from external platforms and feed it through the same indexing pipeline - -The stack: [better-sqlite3](https://github.com/WiseLibs/better-sqlite3) + [sqlite-vec](https://github.com/asg017/sqlite-vec) + [@modelcontextprotocol/sdk](https://github.com/modelcontextprotocol/typescript-sdk) + [@xenova/transformers](https://github.com/xenova/transformers.js) for local embeddings. +- Documents are split into chunks at heading boundaries. +- Each chunk is embedded with the configured provider (local, Ollama or OpenAI). +- Search combines vector similarity ([sqlite-vec](https://github.com/asg017/sqlite-vec)) with FTS5 full-text search. +- Connectors fetch content from other platforms and feed it through the same indexing pipeline. ## Contributing diff --git a/agents.md b/agents.md index 1b21baa8..757c08e6 100644 --- a/agents.md +++ b/agents.md @@ -13,67 +13,66 @@ LibScope is an **AI-powered knowledge base with MCP (Model Context Protocol) int ## Quick Reference — Commands -| Task | Command | -| ----------------------- | ----------------------- | -| Build | `npm run build` | -| Typecheck (no emit) | `npm run typecheck` | -| Run all tests | `npm test` | -| Run tests in watch mode | `npm run test:watch` | -| Run tests with coverage | `npm run test:coverage` | -| Lint | `npm run lint` | -| Lint and auto-fix | `npm run lint:fix` | -| Format check | `npm run format:check` | -| Format and write | `npm run format` | -| Start MCP server | `npm run serve` | -| TypeScript watch | `npm run dev` | +| Task | Command | +| ------------------------- | ----------------------------------- | +| Build | `npm run build` | +| Typecheck (no emit) | `npm run typecheck` | +| Run all tests | `npm test` | +| Run tests in watch mode | `npm run test:watch` | +| Run tests with coverage | `npm run test:coverage` | +| Lint | `npm run lint` | +| Lint and auto-fix | `npm run lint:fix` | +| Format check | `npm run format:check` | +| Format and write | `npm run format` | +| Start MCP server | `npm run serve` | +| TypeScript watch | `npm run dev` | +| Regenerate reference docs | `npm run build && npm run docs:gen` | +| Check reference docs | `npm run docs:check` | **Always run `npm run typecheck` and `npm test` before committing.** ## Architecture +One operation layer, four thin surfaces. Every operation (add, search, ask, get-document, ...) is defined once in `src/core/operations/`: a name, a zod input schema (parameter names, defaults, validation, descriptions), annotations, an optional REST mapping (`http`) and a handler. The CLI, the MCP server, the REST API and the SDK are adapters: they parse input, call `runOperation()` (or `startOperationTask()` for long-running operations) and format the result. + ``` src/ -├── cli/index.ts # CLI entry point (commander). All commands in one file. -├── mcp/server.ts # MCP server (stdio transport, @modelcontextprotocol/sdk) +├── core/ +│ ├── operations/ # The operation layer: types.ts (defineOperation, runOperation, +│ │ # startOperationTask, createOperationContext), schemas.ts (shared +│ │ # input fields), one file per group, index.ts (OPERATIONS) +│ ├── bootstrap.ts # One startup for every surface: config, database, provider, vector table +│ ├── ingest.ts # Add inline content, files, directories, URLs, crawls, repositories +│ ├── search.ts, rag.ts # Hybrid search (vector + FTS5, RRF) and answer() for ask +│ ├── tasks.ts # In-memory background task registry (AbortSignal, progress) +│ ├── overview.ts, document-view.ts, documents.ts, topics.ts, tags.ts, links.ts, packs.ts, ... +│ └── parsers/ # File format parsers (markdown, pdf, docx, html, epub, pptx, csv, yaml, json) +├── cli/ +│ ├── index.ts # Commander program, global options, one error handler +│ └── commands/ # One file per command group; each exports register(program) +├── mcp/ +│ ├── server.ts # createMcpServer(): tools generated from operations (core + admin toolsets) +│ ├── format.ts # Compact text output for tools +│ └── main.ts # Executable entry point (stdio) ├── api/ -│ ├── server.ts # HTTP server bootstrap (createServer, listen) -│ ├── routes.ts # All REST route handlers in one function (~700 lines) -│ ├── middleware.ts # Auth (checkApiKey), rate limiting, body parsing, sendError/sendJson -│ └── openapi.ts # OpenAPI spec generation -├── core/ # Business logic — framework-agnostic, no side effects -│ ├── indexing.ts # Document parsing, chunking by heading, embedding + storage -│ ├── search.ts # Semantic (vector) + FTS5 + LIKE fallback search -│ ├── ratings.ts # Rating storage, aggregation, correction suggestions -│ ├── documents.ts # Document CRUD -│ ├── topics.ts # Topic hierarchy management -│ ├── bulk.ts # Bulk delete/move/retag operations with selector resolution -│ ├── tags.ts # Tag CRUD and document–tag associations -│ ├── dedup.ts # Content deduplication helpers -│ ├── rag.ts # Retrieval-augmented generation (ask a question over the index) -│ ├── url-fetcher.ts # Fetch URL → convert HTML to markdown-like text (SSRF-protected) -│ └── index.ts # Public re-exports (barrel file) -├── connectors/ # External-service sync connectors -│ ├── obsidian.ts # Obsidian vault sync (reads .md files + YAML frontmatter) -│ ├── confluence.ts # Confluence Cloud sync (REST API) -│ ├── notion.ts # Notion sync (official API) -│ ├── onenote.ts # OneNote sync (Microsoft Graph API) -│ ├── slack.ts # Slack channel sync (Web API) -│ ├── sync-tracker.ts # Tracks last-synced state per connector in SQLite -│ ├── http-utils.ts # Authenticated fetch helper shared by connectors (respects allowSelfSignedCerts) -│ └── index.ts # Re-exports -├── db/ -│ ├── connection.ts # SQLite connection + sqlite-vec extension loading -│ ├── schema.ts # Migrations (versioned) + vector table creation -│ └── index.ts # Re-exports -├── providers/ -│ ├── embedding.ts # EmbeddingProvider interface -│ ├── local.ts # all-MiniLM-L6-v2 via @xenova/transformers (384 dims) -│ ├── ollama.ts # Ollama API provider (768 dims default) -│ ├── openai.ts # OpenAI text-embedding-3-small (1536 dims) -│ └── index.ts # Factory function + re-exports -├── config.ts # 3-tier config: env > project .libscope.json > user ~/.libscope/config.json > defaults -├── logger.ts # pino structured logging wrapper -└── errors.ts # Custom error hierarchy +│ ├── routes.ts # API_ROUTES: one route per operation with `http`, plus /openapi.json and /health +│ ├── openapi.ts # buildOpenApiSpec() from the same routes +│ ├── adapter.ts # Request to operation input; response and error mapping +│ ├── middleware.ts # Auth, rate limiting, CORS, body parsing +│ └── server.ts # HTTP server +├── web/ # Dashboard server; dashboard.ts is one large template literal (do not edit) +├── LibScope.ts # SDK: LibScope.create(), add/search/ask/askStream/overview and namespaces +│ # (docs, topics, tags, links, searches, packs, registries, connectors, +│ # tasks, admin, analytics, webhooks) bound to operations +├── lite/ # libscope/lite: createLite() preset, tree-sitter code chunker, normalizeRawInput +├── connectors/ # Notion, Slack, Confluence, Obsidian, OneNote, docs sites; registry.ts lists them +├── registry/ # Git-backed pack registries +├── db/ # better-sqlite3 + sqlite-vec; schema.ts has the migrations (schema version 19) +├── providers/ # Embedding providers: local (384 dims), ollama, openai +├── config-schema.ts # One zod schema: keys, defaults, env var names, docs table +├── config.ts # loadConfig(), secrets.json, config set/unset +├── logger.ts # pino +└── errors.ts # Error hierarchy ``` ## Critical Conventions @@ -109,15 +108,18 @@ All errors extend `LibScopeError` from `src/errors.ts`: LibScopeError (base) ├── DatabaseError ├── EmbeddingError -├── ValidationError +├── ValidationError (REST 400) +├── FetchError (REST 502) ├── ConfigError -├── DocumentNotFoundError -└── ChunkNotFoundError +└── NotFoundError (REST 404; the code names the resource, e.g. LINK_NOT_FOUND) + ├── DocumentNotFoundError + ├── ChunkNotFoundError + └── TopicNotFoundError ``` - Public functions should throw typed errors from this hierarchy, never raw `Error`. -- MCP tool handlers catch errors and return structured error responses. -- CLI shows user-friendly messages; `--verbose` enables full stack traces. +- MCP tools return the error message with `isError: true`. REST answers `{ "error": { "code", "message" } }`. +- The CLI prints `✗ ` and a next step; `--verbose` adds the stack trace. ### Logging @@ -125,7 +127,7 @@ Uses **pino** for structured JSON logging via `src/logger.ts`. - Call `initLogger(level)` once at startup. - Use `getLogger()` everywhere else to obtain the singleton. -- Default level: `info` for CLI, `warn` for MCP server (to avoid polluting stdio). +- The CLI logs nothing by default (`--verbose` or `--log-level` turns logs on). The MCP server logs at `logging.level` to stderr, so stdout carries only JSON-RPC. - Never use `console.log` in `src/core/`, `src/db/`, or `src/providers/`. Use the logger. (`console.log` is acceptable in `src/cli/` for user-facing output.) ### Database @@ -141,7 +143,7 @@ Uses **pino** for structured JSON logging via `src/logger.ts`. 1. Increment `SCHEMA_VERSION` at the top of `src/db/schema.ts`. 2. Add a new key to the `MIGRATIONS` record with the SQL. 3. The migration must insert its version into `schema_version`. -4. Update the schema version assertion in `tests/unit/schema.test.ts`. +4. Update the tests that assert the schema version. ### Embedding Providers @@ -158,11 +160,13 @@ interface EmbeddingProvider { The factory in `src/providers/index.ts` selects the provider based on config. Default is `local` (runs in-process, downloads model on first use). -### Config Precedence +### Configuration + +One zod schema (`src/config-schema.ts`) defines every key, its default, its validation and its environment variable (`LIBSCOPE_
_`, for example `embedding.model` becomes `LIBSCOPE_EMBEDDING_MODEL`). The key table in `docs/reference/configuration.md` is generated from it. -Environment variables > project `.libscope.json` > user `~/.libscope/config.json` > hardcoded defaults. +Precedence: environment variables > `~/.libscope/secrets.json` (API keys only) > project `.libscope.json` > user `~/.libscope/config.json` > defaults. -Env vars: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_OPENAI_API_KEY`, `LIBSCOPE_OLLAMA_URL`, `LIBSCOPE_ALLOW_PRIVATE_URLS`, `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS`. +Main env vars: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_EMBEDDING_MODEL`, `LIBSCOPE_EMBEDDING_URL`, `LIBSCOPE_LLM_PROVIDER`, `LIBSCOPE_LLM_MODEL`, `LIBSCOPE_LLM_URL`, `LIBSCOPE_OPENAI_API_KEY` (or `OPENAI_API_KEY`), `LIBSCOPE_ANTHROPIC_API_KEY` (or `ANTHROPIC_API_KEY`), `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS`, `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS`, `LIBSCOPE_MCP_TOOLSETS`. Not config keys: `LIBSCOPE_WORKSPACE`, `LIBSCOPE_API_KEY` (REST auth), `LIBSCOPE_SECRET_KEY`. ## Security Patterns @@ -205,23 +209,6 @@ const response = await fetch(url, { process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0"; ``` -### SSE streaming — check backpressure - -`res.write()` returns `false` when the socket buffer is full or the client has disconnected. Ignoring the return value wastes compute and holds open connections indefinitely. - -```typescript -// ✅ Correct -for await (const event of stream) { - const ok = res.write(`data: ${JSON.stringify(event)}\n\n`); - if (!ok) break; -} - -// ❌ Wrong — no backpressure handling -for await (const event of stream) { - res.write(`data: ${JSON.stringify(event)}\n\n`); -} -``` - ## Testing ### Framework @@ -233,28 +220,20 @@ for await (const event of stream) { ``` tests/ ├── fixtures/ -│ ├── mock-provider.ts # Deterministic 4D embedding provider (use in all unit tests) -│ ├── test-db.ts # In-memory SQLite with migrations (no sqlite-vec) -│ ├── sample-api-docs.md # Sample library documentation -│ └── sample-topic-docs.md # Sample topic documentation -├── unit/ # Fast, isolated, no I/O or network -│ ├── chunking.test.ts -│ ├── config.test.ts -│ ├── documents.test.ts -│ ├── errors.test.ts -│ ├── ratings.test.ts -│ ├── schema.test.ts -│ └── topics.test.ts -└── integration/ - └── workflow.test.ts # Full index → search → rate → query flow +│ ├── mock-provider.ts # Deterministic 4D embedding provider (use in all unit tests) +│ ├── test-db.ts # createTestDb(): in-memory SQLite with migrations; createTestDbWithVec() +│ ├── helpers.ts # insertDoc, insertChunk, seedTestDocument +│ └── sample-*.md # Sample documents +├── unit/ # Fast, isolated, no network (unit/operations/ tests the operations) +└── integration/ # Real SQLite, full workflows (index, search, rate) ``` ### Writing Tests - **Use `MockEmbeddingProvider`** from `tests/fixtures/mock-provider.ts` for all tests that need embeddings. It returns deterministic 4D vectors — no model download, no network. - **Use `createTestDb()`** from `tests/fixtures/test-db.ts` for an in-memory SQLite instance with all migrations applied. -- **sqlite-vec is NOT available in tests.** The test DB is plain SQLite. Vector search tests exercise the FTS5/LIKE fallback path. This is by design. -- **Coverage thresholds** (enforced in `vitest.config.ts`): statements ≥ 75%, branches ≥ 74%, functions ≥ 75%, lines ≥ 75%. CLI code (`src/cli/`) is excluded from coverage. +- **`createTestDb()` does not load sqlite-vec.** Vector search tests use `createTestDbWithVec()` or exercise the FTS5/keyword path. +- **Coverage thresholds** (enforced in `vitest.config.ts`): statements ≥ 75%, branches ≥ 74%, functions ≥ 75%, lines ≥ 75%. `src/cli/`, `src/mcp/main.ts` and the network embedding providers are excluded from coverage. - **Always run `npm run test:coverage`** (not just `npm test`) before pushing. CI runs `test:coverage`, which fails if any threshold is missed. `npm test` alone does NOT check coverage. - When adding new source files, ensure adequate test coverage so global thresholds are not violated. New files with many uncovered branches will drag the overall percentage down. - Tests should be fast (< 1 second total), deterministic, and not depend on ordering. @@ -276,25 +255,23 @@ SQLite `datetime('now')` has **second-level precision**. If you insert multiple ## CI/CD -Three GitHub Actions workflows in `.github/workflows/`: +Workflows in `.github/workflows/`: -| Workflow | Trigger | What it does | -| ------------- | ------------- | --------------------------------------------------- | -| `ci.yml` | Push & PR | Lint, typecheck, test (Node 18/20/22 matrix), build | -| `release.yml` | Tags `v*.*.*` | Full CI then `npm publish --provenance` | -| `codeql.yml` | Weekly + PR | CodeQL security scanning | +| Workflow | What it does | +| ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | +| `ci.yml` | Lint, format check, typecheck, `test:coverage` (Node 20 and 22), build, `docs:check` (the generated reference docs must be current) | +| `codeql.yml` | CodeQL security scanning | +| `release-please.yml` | Release PRs and the changelog | +| `docker.yml` | Docker image | +| `sdk-python.yml`, `sdk-go.yml` | Python and Go client SDKs | +| `release-python.yml` | Python SDK release | -## MCP Server +## Surfaces -The MCP server (`src/mcp/server.ts`) uses **stdio transport** and exposes 5 tools: - -1. `search-docs` — Semantic search with optional topic/library/version/rating filters -2. `get-document` — Retrieve a document by ID -3. `rate-document` — Rate a document or suggest corrections -4. `submit-document` — Submit a new document for indexing -5. `list-topics` — List available topics - -To test the MCP server locally: `npm run build && npm run serve` +- **MCP** (`src/mcp/server.ts`): 11 core tools (`search`, `ask`, `get-document`, `list-documents`, `overview`, `submit-document`, `update-document`, `delete-document`, `rate-document`, `link-documents`, `task`). `ask` is registered only with passthrough or a configured LLM. The admin toolset (`mcp.toolsets` or `LIBSCOPE_MCP_TOOLSETS=admin`) adds `sync`, `install-pack`, `list-packs` and `reindex-documents`. See `docs/reference/mcp-tools.md`. Test locally: `npm run build && npm run serve`. +- **CLI** (`src/cli/commands/*`): see `docs/reference/cli.md` or `libscope --help`. +- **REST** (`src/api/routes.ts`): every route under `/api/v1` comes from an operation's `http` mapping. Long-running operations answer `202` with a task ID. See `docs/reference/rest-api.md`. +- **SDK** (`src/LibScope.ts`): methods call the same operations. The package root exports only the class, its types, the provider interfaces and the errors. ## Parallel Agent Work — Git Worktrees @@ -367,9 +344,9 @@ Every PR must follow this complete lifecycle. **Do not consider a PR done until ## Adding a New Feature — Checklist -1. Add business logic in `src/core/` (no framework dependencies). +1. Add business logic in `src/core/` (no surface dependencies). 2. If it needs new DB tables/columns, add a migration in `src/db/schema.ts`. -3. Expose via MCP tool in `src/mcp/server.ts` and/or CLI command in `src/cli/index.ts`. +3. Add or extend an operation in `src/core/operations/` (an input schema with `.describe()` on every field, defaults only in the schema, an `http` mapping for REST). The REST route and the OpenAPI entry follow from it. Add it to an SDK namespace in `src/LibScope.ts`, a CLI command in `src/cli/commands/`, and, only if assistants need it, an MCP tool in `src/mcp/server.ts`. 4. Write unit tests in `tests/unit/` using `MockEmbeddingProvider` and `createTestDb()`. 5. Add integration coverage in `tests/integration/workflow.test.ts` if it's a core flow. 6. Run `npm run typecheck && npm run test:coverage && npm run lint` — all must pass. **Use `test:coverage`, not `test`** — CI enforces coverage thresholds and will fail if new code drops coverage below the configured minimums (see `vitest.config.ts` thresholds). @@ -380,21 +357,17 @@ Every PR must follow this complete lifecycle. **Do not consider a PR done until ## Documentation -Every user-facing change **must** update all relevant documentation. Documentation lives in multiple places — check each one: - -| Location | What it covers | -| --------------------------------- | ------------------------------------------------------------------ | -| `README.md` | Top-level overview, quickstart, config tables, CLI summary | -| `docs/guide/getting-started.md` | First-run walkthrough | -| `docs/guide/configuration.md` | Config guide with env var table and examples | -| `docs/reference/cli.md` | Full CLI command reference | -| `docs/reference/configuration.md` | Complete config key reference, env vars, example config | -| `agents.md` | Agent/Copilot guide — architecture, conventions, config precedence | +Every user-facing change **must** update the documentation. The CLI, MCP tool, config key and REST route tables in `docs/reference/` are generated from the code: run `npm run build && npm run docs:gen` and commit the result. CI runs `npm run docs:check`, which fails when a generated block is out of date. -**What to update for common change types:** +| Location | What it covers | +| --------------------------------- | ---------------------------------------------------------------------------- | +| `README.md` | Pitch and quick start (CLI, MCP, SDK); links to the docs site | +| `docs/guide/*.md` | Guides (getting started, configuration, connectors, programmatic usage, ...) | +| `docs/reference/cli.md` | CLI reference (generated option tables, hand-written examples) | +| `docs/reference/mcp-tools.md` | MCP tools (generated) | +| `docs/reference/rest-api.md` | REST routes (generated) and conventions | +| `docs/reference/configuration.md` | Config keys (generated), config files, other env vars | +| `docs/migration-v2.md` | Every name removed or renamed in 2.0 | +| `agents.md`, `CLAUDE.md` | Agent guides: architecture, conventions | -- **New config key:** `src/config.ts` (interface + defaults + env override) → `README.md` (env var table, example config) → `docs/guide/configuration.md` (env var table) → `docs/reference/configuration.md` (config keys table, env vars table, example config) → `agents.md` (config precedence env var list) -- **New CLI command:** `src/cli/index.ts` → `README.md` (CLI table) → `docs/reference/cli.md` (command reference) -- **New MCP tool:** `src/mcp/server.ts` → `README.md` (MCP tools list) → `agents.md` (MCP Server section) -- **New connector:** `src/connectors/` → `README.md` (connectors section) → `docs/guide/` (new guide page) → `docs/reference/cli.md` (sync/disconnect commands) -- **New env var:** All env var tables: `README.md`, `docs/guide/configuration.md`, `docs/reference/configuration.md` +Do not edit `CHANGELOG.md` or `docs/changelog.md`; release-please writes them. diff --git a/docker-compose.yml b/docker-compose.yml index 720cc75e..911be0e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,13 +2,12 @@ services: libscope: build: . ports: - - "3420:3420" # API - - "3421:3421" # Web UI + - "3378:3378" # REST API volumes: - libscope-data:/data environment: - - LIBSCOPE_DATA_DIR=/data - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - LIBSCOPE_API_KEY=${LIBSCOPE_API_KEY:-} restart: unless-stopped # Optional: Ollama for local embeddings diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 350a972f..46d211a7 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -33,6 +33,10 @@ export default defineConfig({ text: "Changelog", link: "/changelog", }, + { + text: "Migrating to 2.0", + link: "/migration-v2", + }, { text: "Contributing", link: "/contributing", @@ -48,6 +52,7 @@ export default defineConfig({ items: [ { text: "Getting Started", link: "/guide/getting-started" }, { text: "Configuration", link: "/guide/configuration" }, + { text: "Migrating to 2.0", link: "/migration-v2" }, ], }, { diff --git a/docs/contributing.md b/docs/contributing.md index f1b9e5aa..c075e0b4 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -8,44 +8,56 @@ Thanks for your interest in contributing! Here's how to get started. git clone https://github.com/RobertLD/libscope.git cd libscope npm install -npm run build && node dist/cli/index.js init +npm run build && node dist/cli/index.js doctor --fix npm run dev # TypeScript watch mode ``` ## Scripts -| Command | Description | -| ----------------------- | ----------------------------- | -| `npm run build` | Compile TypeScript to `dist/` | -| `npm run dev` | Watch mode compilation | -| `npm run lint` | Run ESLint | -| `npm run lint:fix` | ESLint with auto-fix | -| `npm run format` | Format with Prettier | -| `npm run format:check` | Check formatting | -| `npm run typecheck` | Type-check without emitting | -| `npm test` | Run tests | -| `npm run test:watch` | Tests in watch mode | -| `npm run test:coverage` | Tests with coverage report | +| Command | Description | +| ----------------------- | ----------------------------------------------------------------------- | +| `npm run build` | Compile TypeScript to `dist/` | +| `npm run dev` | Watch mode compilation | +| `npm run lint` | Run ESLint | +| `npm run lint:fix` | ESLint with auto-fix | +| `npm run format` | Format with Prettier | +| `npm run format:check` | Check formatting | +| `npm run typecheck` | Type-check without emitting | +| `npm test` | Run tests | +| `npm run test:watch` | Tests in watch mode | +| `npm run test:coverage` | Tests with coverage report | +| `npm run docs:gen` | Update the generated blocks in `docs/reference` (after `npm run build`) | +| `npm run docs:check` | Fail when a generated block is out of date (runs in CI) | +| `npm run docs:dev` | Docs site dev server (VitePress) | +| `npm run docs:build` | Build the docs site | ## Project Structure ``` src/ -├── core/ # Business logic (indexing, search, ratings, topics, documents) -├── db/ # SQLite schema, migrations, connection management +├── core/ +│ ├── operations/ # The operation layer: every command, tool, route and SDK method calls one +│ ├── parsers/ # File format parsers +│ └── ... # Business logic (ingest, indexing, search, rag, documents, tasks, ...) +├── cli/ # CLI: index.ts and one file per command group in commands/ +├── mcp/ # MCP server: tools generated from operations +├── api/ # REST API: routes generated from operations, OpenAPI +├── web/ # Web dashboard +├── lite/ # libscope/lite: createLite(), code chunker, normalizeRawInput() +├── db/ # SQLite schema, migrations, connection ├── providers/ # Embedding providers (local, Ollama, OpenAI) -├── mcp/ # MCP server and tool definitions -├── cli/ # CLI entry point and commands -├── api/ # REST API server and routes -├── web/ # Web dashboard and knowledge graph -├── connectors/ # Third-party syncs (Obsidian, Notion, Confluence, Slack, OneNote) -├── registry/ # Git-backed pack registry system -├── config.ts # Configuration management +├── connectors/ # Notion, Slack, Confluence, Obsidian, OneNote, doc sites +├── registry/ # Git pack registries +├── config-schema.ts # Config schema (keys, defaults, env vars) +├── config.ts # Config loading +├── LibScope.ts # Node.js SDK class ├── logger.ts # Structured logging (pino) -└── errors.ts # Custom error hierarchy +└── errors.ts # Error hierarchy +scripts/ +└── gen-docs.mjs # Fills the generated blocks of docs/reference tests/ ├── unit/ # Fast isolated tests with mocked dependencies -├── integration/ # Tests with real SQLite DB +├── integration/ # Tests with a real SQLite database └── fixtures/ # Test helpers, mock providers, sample data ``` @@ -53,9 +65,9 @@ For a full module-by-module breakdown and data flow diagrams, see the [Architect ## Design Principles -**Entry points are thin.** The CLI, MCP server, and REST API are adapters. They parse input and format output. All business logic lives in `src/core/`. +**Operations first.** New functionality is an operation in `src/core/operations/` (a zod input schema and a handler). The CLI, MCP server, REST API and SDK are thin adapters over the operations: they parse input and format output. Business logic lives in `src/core/`. See [How to Add an Operation](/guide/architecture#how-to-add-an-operation). -**Errors are typed.** Use the appropriate `LibScopeError` subclass (`DatabaseError`, `ValidationError`, `FetchError`, etc.) rather than throwing plain `Error`. MCP tool handlers must be wrapped with `withErrorHandling()` from `src/mcp/errors.ts`. +**Errors are typed.** Use the appropriate `LibScopeError` subclass (`DatabaseError`, `ValidationError`, `FetchError`, etc.) rather than throwing plain `Error`. Each surface maps errors in one place (`src/cli/errors.ts`, `withErrorHandling()` in `src/mcp/errors.ts`, `describeError()` in `src/api/adapter.ts`). **Core modules are testable.** They accept `db` and `provider` as parameters — never import them directly inside a function. This makes it easy to swap in `createTestDb()` and `MockEmbeddingProvider` in tests. @@ -63,6 +75,8 @@ For a full module-by-module breakdown and data flow diagrams, see the [Architect **No `any`.** Use `unknown` and narrow with type guards. The ESLint rule `no-explicit-any: "error"` is enforced. +**Generated references.** The CLI, MCP tool, configuration and REST references are partly generated from the code. After you change a command, tool, config key or route, run `npm run build && npm run docs:gen` and commit the result. + ## Making Changes 1. **Create a branch** from `main` @@ -70,7 +84,8 @@ For a full module-by-module breakdown and data flow diagrams, see the [Architect 3. **Add tests** for new functionality 4. **Run the full check suite:** ```bash - npm run lint && npm run typecheck && npm test + npm run format:check && npm run lint && npm run typecheck && npm test + npm run build && npm run docs:check ``` 5. **Commit** with [Conventional Commits](https://www.conventionalcommits.org/): - `feat: add bulk import from directory` @@ -104,4 +119,4 @@ Open an issue on GitHub with: ## License -By contributing, you agree that your contributions will be licensed under the MIT License. +By contributing, you agree that your contributions will be licensed under the project's license (Business Source License 1.1, see `LICENSE`). diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 006e5be1..ff26c820 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -1,367 +1,375 @@ # Architecture -This guide explains how LibScope is structured internally. It is intended for contributors and developers who want to understand or extend the codebase. +This guide explains how LibScope is structured internally. It is for contributors and developers who want to understand or extend the code. ## System Layers -LibScope is organized into four distinct layers: - ``` -┌─────────────────────────────────────────────────────────┐ -│ Entry Points │ -│ CLI (Commander.js) MCP Server REST API LibScopeLite │ -└──────────────────────────┬──────────────────────────────┘ - │ -┌──────────────────────────▼──────────────────────────────┐ -│ Core Business Logic │ -│ indexing · search · rag · documents · parsers · … │ -└──────────────────────────┬──────────────────────────────┘ - │ -┌──────────────────────────▼──────────────────────────────┐ -│ Infrastructure │ -│ db/ (SQLite + sqlite-vec) providers/ (embeddings)│ -└─────────────────────────────────────────────────────────┘ +┌──────────────────────────────────────────────────────────────────┐ +│ Surfaces (adapters) │ +│ CLI MCP server REST API Web dashboard SDK (LibScope) │ +└────────────────────────────────┬─────────────────────────────────┘ + │ runOperation / startOperationTask +┌────────────────────────────────▼─────────────────────────────────┐ +│ Operation layer (src/core/operations) │ +│ name + zod input schema + handler, one file per group │ +└────────────────────────────────┬─────────────────────────────────┘ + │ +┌────────────────────────────────▼─────────────────────────────────┐ +│ Core logic (src/core) │ +│ ingest · indexing · search · rag · documents · tasks · parsers │ +└────────────────────────────────┬─────────────────────────────────┘ + │ +┌────────────────────────────────▼─────────────────────────────────┐ +│ Infrastructure │ +│ db/ (SQLite + sqlite-vec) providers/ (embeddings) connectors│ +└──────────────────────────────────────────────────────────────────┘ ``` -**Entry points** (`src/cli/`, `src/mcp/`, `src/api/`, `src/lite/`) are thin adapters. They parse input, call core functions, and format output. They contain no business logic. +**Operations** (`src/core/operations/`) are the single entry into the core. An operation has a kebab-case `name`, a `group`, a one-line `summary`, a zod `input` schema (every field has a description and its default), optional `annotations` (`readOnly`, `destructive`, `idempotent`, `longRunning`), an optional `http` mapping (method and path), and a `handler(ctx, input)`. The handler receives an `OperationContext` with the database, the embedding provider, the config, the calling surface, an optional `AbortSignal` and progress callback, and the LLM. -**Core** (`src/core/`) contains all business logic. Core modules are plain TypeScript functions — they don't know whether they were called from the CLI, an MCP tool, the REST API, or `LibScopeLite`. +**Surfaces** are thin adapters. They parse their input, call `runOperation(op, ctx, input)` (or `startOperationTask` for a background task), and format the result. They contain no business logic. Because every surface uses the same schema, parameter names, defaults, validation and results are the same in the CLI, the MCP server, the REST API and the SDK. -**Infrastructure** (`src/db/`, `src/providers/`) handles persistence and external services. The database layer uses better-sqlite3 (synchronous). The provider layer abstracts embedding models behind a common interface. +**Core** (`src/core/`) contains the business logic as plain functions over a database and a provider. -### LibScope Lite Layer +**Infrastructure** (`src/db/`, `src/providers/`, `src/connectors/`, `src/registry/`) handles storage, embedding models, third-party services and git pack registries. -`src/lite/` is a separate entry point that exposes a minimal embeddable API built on top of the same core and infrastructure modules: +### The operation layer -``` -libscope/lite → src/lite/index.ts → LibScopeLite class - ├── core/indexing.ts - ├── core/search.ts - ├── core/rag.ts - ├── core/ratings.ts - ├── db/connection.ts - └── providers/ -``` +| File | Content | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `types.ts` | `defineOperation`, `runOperation`, `startOperationTask`, `parseOperationInput`, `createOperationContext`, `OperationContext`, `ListResult` | +| `schemas.ts` | Shared input fields (`documentId`, `topic`, `sourceType`, `tags`, `limit`, `offset`, ...) so every operation uses the same names | +| `index.ts` | `OPERATIONS` (every operation, in display order) and `getOperation(name)` | +| `documents.ts`, `search.ts`, `links.ts` | One file per group: documents, search and ask, links | +| `graph.ts`, `tags.ts`, `topics.ts` | Knowledge graph, tags, topics | +| `searches.ts`, `packs.ts` | Saved searches, knowledge packs | +| `registries.ts`, `connectors.ts` | Git pack registries, saved connector connections | +| `admin.ts`, `analytics.ts` | Overview, reindex, dedupe, backup/restore, prune, bulk changes; search analytics | +| `webhooks.ts`, `tasks.ts` | Webhooks, background tasks | + +`runOperation` validates the input (a zod error becomes a `ValidationError` that names the operation and the field), runs the operation's `validate` hook, and calls the handler. `startOperationTask` does the same validation at once, then runs the handler as a background task in the shared task registry (`src/core/tasks.ts`) and returns the task. + +### Surfaces + +| Surface | Files | How it uses the operations | +| --------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CLI | `src/cli/index.ts`, `src/cli/commands/*.ts` | Each command file exports `register(program)`. A command maps its arguments and flags to the operation input and calls `run(op, input, format)` from `src/cli/run.ts`, which prints JSON with `--json` or the human format. `src/cli/errors.ts` is the one error handler. `program` is exported for tests and docs. | +| MCP server | `src/mcp/server.ts`, `src/mcp/main.ts`, `src/mcp/format.ts` | `createMcpServer()` registers each tool from an operation: the input schema, the description (the summary) and the annotations come from the operation. It registers the 11 core tools, and the admin toolset when `mcp.toolsets` enables it. `main.ts` runs it on stdio. Importing `server.ts` starts nothing. | +| REST API | `src/api/routes.ts`, `src/api/adapter.ts`, `src/api/openapi.ts` | `API_ROUTES` has one route per operation that declares `http`, plus `/openapi.json` and `/api/v1/health`. A `longRunning` operation answers `202` with a task ID. `buildOpenApiSpec(API_ROUTES)` generates the OpenAPI 3.1 document from the input schemas. | +| Web dashboard | `src/web/server.ts`, `src/web/dashboard.ts` | The dashboard's JSON routes run operations and reshape the results for the dashboard page. | +| SDK | `src/LibScope.ts`, `src/core/index.ts` | `LibScope` has top-level methods (`add`, `search`, `ask`, `askStream`, `overview`) and namespaces (`docs`, `topics`, `tags`, `links`, `searches`, `packs`, `registries`, `connectors`, `tasks`, `admin`, `analytics`, `webhooks`). The `NAMESPACES` table binds each method to an operation, and the types come from its schema. | +| `libscope/lite` | `src/lite/` | `createLite()` is a preset of `LibScope` that does not read config files. It also exports a tree-sitter code chunker and `normalizeRawInput()`. It does not import the CLI, the MCP server or the connectors. | -`LibScopeLite` deliberately omits connectors, topics, packs, webhooks, and registry — keeping the API surface small and the import footprint minimal for embedding in external applications. +### Startup + +Every surface starts with `bootstrap()` (`src/core/bootstrap.ts`): load the config, resolve the database path (explicit path, `database.path`, or the workspace database), open and migrate the database, create the embedding provider, and create or check the vector table. `createOperationContext()` then builds the context for the surface. + +### Shared core pieces + +| File | Purpose | +| ---------------------------- | --------------------------------------------------------------------------------------------------------- | +| `src/core/ingest.ts` | One `add` path for inline content, files, directories, URLs (optionally crawled) and repositories | +| `src/core/tasks.ts` | Background task registry with `AbortSignal` and progress; tasks are kept in memory for one hour | +| `src/core/overview.ts` | Counts, topics, installed packs, index identity and health (`overview` operation, `admin stats`) | +| `src/core/document-view.ts` | A document with its tags, links and ratings, with content paging (`get-document`) | +| `src/core/rag.ts` | `answer()`: LLM answer or passthrough context; LLM providers | +| `src/connectors/registry.ts` | One table of connector types: config shape, secret fields, sync and disconnect; schedules for `serve api` | +| `src/config-schema.ts` | The zod config schema: every key, its default, its env vars, and the generated config reference | ## Module Map ``` src/ ├── cli/ -│ ├── index.ts # main CLI entry (#!/usr/bin/env node) -│ └── commands/ # each file exports registerCommands(program) +│ ├── index.ts # program, global options, error handler; register() per command file +│ ├── commands/ # add, search (+ ask), docs, topics (+ tags), searches, bulk, connectors, +│ │ # pack, registry, serve, config, workspace, webhooks, admin, doctor +│ ├── run.ts # run an operation and print JSON or the human format +│ ├── context.ts # the command's OperationContext (opened on first use) +│ ├── options.ts # shared flags (document filters) and parsing helpers +│ ├── errors.ts # "✗ message" + next-step hint; stack only with --verbose +│ ├── reporter.ts # progress line on stderr +│ ├── repl.ts # interactive search (search with no query) +│ └── confirm.ts # confirmation prompts (-y skips them) ├── mcp/ -│ ├── server.ts # MCP entry point — registers all tools -│ ├── tools/ # each file exports registerTools(server, db, provider) -│ └── errors.ts # withErrorHandling() wrapper for MCP tool handlers +│ ├── server.ts # createMcpServer(), runStdioServer(); tools generated from operations +│ ├── main.ts # executable: MCP server on stdio +│ ├── format.ts # compact text output of each tool +│ └── errors.ts # withErrorHandling(): errors become isError results ├── api/ -│ ├── server.ts # Express app factory -│ ├── routes/ # route handlers -│ └── openapi.ts # OpenAPI 3.0 spec +│ ├── server.ts # startApiServer(): node:http server, rate limit, auth, schedules +│ ├── routes.ts # API_ROUTES generated from the operations; handleRequest() +│ ├── adapter.ts # request -> operation input; operation result/error -> HTTP response +│ ├── openapi.ts # buildOpenApiSpec(): OpenAPI 3.1 from the routes +│ └── middleware.ts # CORS, API key, rate limit, security headers, JSON responses ├── web/ -│ ├── server.ts # HTTP server (port 3377) -│ ├── dashboard.ts # self-contained dashboard HTML -│ └── graph-api.ts # knowledge graph data API +│ ├── server.ts # dashboard server (port 3377) and its JSON routes +│ └── dashboard.ts # dashboard page and scripts ├── core/ -│ ├── indexing.ts # document chunking and embedding +│ ├── operations/ # the operation layer (see above) +│ ├── bootstrap.ts # one startup sequence for every surface +│ ├── ingest.ts # add content, files, directories, URLs, repositories +│ ├── indexing.ts # chunking and embedding │ ├── search.ts # hybrid vector + FTS5 search -│ ├── rag.ts # LLM-based question answering +│ ├── rag.ts # ask: LLM answer or passthrough context │ ├── documents.ts # document CRUD +│ ├── document-view.ts # document with tags, links, ratings; content paging +│ ├── overview.ts # counts, topics, packs, index, health +│ ├── tasks.ts # background task registry │ ├── versioning.ts # document version history │ ├── topics.ts # topic hierarchy -│ ├── tags.ts # tag management and auto-suggest -│ ├── links.ts # cross-document references -│ ├── ratings.ts # document/chunk ratings +│ ├── tags.ts # tags and tag suggestions +│ ├── links.ts # typed links between documents +│ ├── ratings.ts # document and chunk ratings │ ├── dedup.ts # duplicate detection -│ ├── bulk.ts # bulk operations +│ ├── bulk.ts # bulk delete, retag, move │ ├── analytics.ts # search analytics, knowledge gaps -│ ├── graph.ts # knowledge graph with cluster detection -│ ├── packs.ts # knowledge pack create/install -│ ├── batch.ts # parallel batch import -│ ├── batch-search.ts # concurrent multi-query search -│ ├── url-fetcher.ts # HTTP fetch with retry, proxy, cert handling -│ ├── spider.ts # recursive web crawler -│ ├── link-extractor.ts # extract links from HTML/Markdown/Wikilinks -│ ├── repo.ts # GitHub/GitLab repo cloning and indexing -│ ├── watcher.ts # file system watching for auto-reindex -│ ├── reindex.ts # re-embedding after provider switch -│ ├── scheduler.ts # background task scheduling -│ ├── webhooks.ts # event webhooks with HMAC signing -│ ├── saved-searches.ts # named query persistence -│ ├── workspace.ts # workspace isolation -│ ├── export.ts # full knowledge base backup/restore -│ ├── ttl.ts # document expiry management -│ └── parsers/ # file format parsers -│ ├── markdown.ts # Markdown + MDX -│ ├── text.ts # plain text -│ ├── pdf.ts # PDF (optional: pdf-parse) -│ ├── word.ts # DOCX (optional: mammoth) -│ ├── epub.ts # EPUB (optional: epub2) -│ ├── pptx.ts # PowerPoint (optional: pizzip) -│ ├── html.ts # HTML -│ ├── csv.ts # CSV -│ ├── json-parser.ts # JSON -│ └── yaml.ts # YAML +│ ├── graph.ts # knowledge graph with clusters +│ ├── packs.ts # knowledge pack create/install/remove +│ ├── url-fetcher.ts # HTTP fetch with retry, proxy and certificate handling +│ ├── spider.ts # web crawler +│ ├── link-extractor.ts # links from HTML, Markdown and wikilinks +│ ├── repo.ts # GitHub/GitLab repository indexing +│ ├── watcher.ts # file watching for `add --watch` +│ ├── reindex.ts # re-embed chunks, rebuild the vector index +│ ├── scheduler.ts # cron schedules of saved connections +│ ├── webhooks.ts # webhooks with HMAC signing +│ ├── events.ts # document and search events (webhooks subscribe to them) +│ ├── saved-searches.ts # saved searches +│ ├── workspace.ts # workspaces +│ ├── export.ts # backup and restore +│ ├── ttl.ts # expiry of documents (prune-expired) +│ ├── index.ts # package root exports +│ └── parsers/ # markdown, text, pdf, word, epub, pptx, html, csv, json, yaml ├── db/ -│ ├── connection.ts # SQLite connection factory (WAL mode, sqlite-vec) -│ └── schema.ts # SCHEMA_VERSION, MIGRATIONS, createSchema() +│ ├── connection.ts # database path resolution, SQLite connection (WAL, sqlite-vec) +│ ├── schema.ts # SCHEMA_VERSION, MIGRATIONS, runMigrations(), createVectorTable() +│ ├── index-meta.ts # embedding model and vector size recorded for the vector index +│ └── validate.ts # row validation helpers ├── providers/ -│ ├── index.ts # EmbeddingProvider interface + factory +│ ├── embedding.ts # EmbeddingProvider interface +│ ├── index.ts # createEmbeddingProvider() from config +│ ├── dimensions.ts # vector sizes of known models │ ├── local.ts # @xenova/transformers (all-MiniLM-L6-v2) │ ├── ollama.ts # Ollama HTTP API │ └── openai.ts # OpenAI embeddings API -├── registry/ -│ ├── types.ts # RegistryEntry, PackSummary, PackManifest -│ ├── config.ts # registry list in ~/.libscope/config.json -│ ├── git.ts # git clone/pull/commit/push -│ ├── sync.ts # registry syncing with auto-sync intervals -│ ├── search.ts # pack search across registries -│ ├── resolve.ts # pack resolution (version conflicts) -│ ├── publish.ts # publishing packs to registries -│ └── checksum.ts # SHA-256 verification +├── registry/ # git pack registries: config, git, sync, search, resolve, publish, checksum ├── connectors/ -│ ├── obsidian.ts # Obsidian vault sync -│ ├── notion.ts # Notion API sync -│ ├── confluence.ts # Confluence API sync -│ ├── slack.ts # Slack API sync -│ ├── onenote.ts # Microsoft Graph API sync -│ ├── http-utils.ts # shared retry logic with exponential backoff -│ └── sync-tracker.ts # sync history and status in database +│ ├── registry.ts # connector registry (types, secrets, sync, disconnect, schedules) +│ ├── saved-config.ts # saved connections in ~/.libscope/connectors/.json +│ ├── notion.ts, slack.ts, confluence.ts, obsidian.ts, onenote.ts, docs.ts +│ ├── http-utils.ts # retry with exponential backoff +│ └── sync-tracker.ts # sync history and status in the database ├── lite/ -│ ├── index.ts # public entrypoint — exports LibScopeLite + types -│ ├── core.ts # LibScopeLite class implementation -│ ├── types.ts # LiteOptions, LiteDoc, LiteSearchResult, etc. -│ ├── normalize.ts # raw input → markdown (dispatches to core/parsers/) -│ └── chunker-treesitter.ts # optional tree-sitter code chunker (TS/JS/Python) -├── config.ts # loadConfig() — merges env, project, user, defaults +│ ├── index.ts # libscope/lite: package root + createLite, chunker, normalizeRawInput +│ ├── core.ts # createLite(), createCodeChunker() +│ ├── normalize.ts # raw input -> { title, content } (uses core/parsers) +│ └── chunker-treesitter.ts # optional tree-sitter code chunker +├── utils/ # glob, retry, row validation +├── config-schema.ts # zod config schema (keys, defaults, env vars, docs table) +├── config.ts # loadConfig(), config files, secrets.json, config get/set ├── errors.ts # LibScopeError hierarchy -├── logger.ts # pino logger with child logger support -└── LibScope.ts # main public API class +├── logger.ts # pino logger +└── LibScope.ts # the SDK class ``` ## Key Design Patterns ### Error Hierarchy -All errors extend `LibScopeError`, which carries a `code` string and an optional `cause`: - -```typescript -// src/errors.ts -class LibScopeError extends Error { - constructor(message: string, public readonly code: string, options?: ErrorOptions) -} +All errors extend `LibScopeError`, which has a `code` string and an optional `cause`: -// Subclasses: -DatabaseError // SQLite failures -EmbeddingError // provider failures -ValidationError // bad input -FetchError // HTTP fetch failures -ConfigError // misconfiguration -DocumentNotFoundError -ChunkNotFoundError -TopicNotFoundError +``` +LibScopeError + ├── DatabaseError + ├── EmbeddingError + ├── ValidationError # bad input; REST 400 + ├── FetchError # a fetched URL failed; REST 502 + ├── ConfigError # misconfiguration, with the setting to change + └── NotFoundError # REST 404; code names the resource (LINK_NOT_FOUND, ...) + ├── DocumentNotFoundError + ├── ChunkNotFoundError + └── TopicNotFoundError ``` -Always throw the most specific subclass. MCP tool handlers must be wrapped with `withErrorHandling()` from `src/mcp/errors.ts` — this converts `LibScopeError` to well-structured MCP error responses. +Throw the most specific class. The surfaces map errors in one place each: `src/cli/errors.ts`, `withErrorHandling()` in `src/mcp/errors.ts`, and `describeError()` in `src/api/adapter.ts`. ### Embedding Provider Interface -All embedding providers implement the same interface: - ```typescript +// src/providers/embedding.ts interface EmbeddingProvider { - readonly dimensions: number; - embed(texts: string[]): Promise; + readonly name: string; + readonly model?: string | undefined; + readonly dimensions: number; // 0 until the first embedding when unknown + embed(text: string): Promise; + embedBatch(texts: string[]): Promise; } ``` -The factory function in `src/providers/index.ts` returns the correct implementation based on config. Core modules accept an `EmbeddingProvider` and never import a specific provider directly — this makes them testable with the `MockEmbeddingProvider` fixture. +`createEmbeddingProvider(config)` in `src/providers/index.ts` returns the configured provider. Core modules receive a provider and never import a specific one, so tests use the `MockEmbeddingProvider` fixture. The vector index records the provider, model and vector size (`src/db/index-meta.ts`); a mismatch is a `ConfigError` that suggests `libscope admin reindex --rebuild`. ### Database Migrations -Schema migrations in `src/db/schema.ts` follow this pattern: +Migrations are in `src/db/schema.ts`. The current schema version is 19. ```typescript -export const SCHEMA_VERSION = 17; +const SCHEMA_VERSION = 19; -export const MIGRATIONS: Record = { - 1: "CREATE TABLE ...", +const MIGRATIONS: Record = { // ... - 17: "ALTER TABLE chunks ADD COLUMN ...", + 19: "...", }; ``` -To add a migration: increment `SCHEMA_VERSION` and add a new entry in `MIGRATIONS` with the new version number as the key. `createSchema()` runs all missing migrations on startup. +To add a migration, increment `SCHEMA_VERSION` and add an entry to `MIGRATIONS` with the new version number as its key. `runMigrations()` applies the missing migrations at startup. -### Configuration Merging +### Configuration -`loadConfig()` in `src/config.ts` merges four tiers, highest priority first: +`src/config-schema.ts` defines every config key once (type, default, environment variables, secret or not, description). `loadConfig()` in `src/config.ts` merges, highest priority first: -1. Environment variables (`LIBSCOPE_*`) -2. Project `.libscope.json` (current working directory) +1. Environment variables (`LIBSCOPE_
_`) +2. Project `.libscope.json` 3. User `~/.libscope/config.json` -4. Defaults - -The result is cached for 30 seconds to avoid repeated file reads. Configuration is passed down to core modules — they never read environment variables directly. +4. Schema defaults -### Connectors Pattern +API keys come only from environment variables and `~/.libscope/secrets.json`. The result is cached for 30 seconds. The same schema drives `libscope config get/set/unset` and the [configuration reference](/reference/configuration). -Each connector in `src/connectors/` follows the same structure: +### Connectors -1. **Auth config** stored in `~/.libscope/connectors/.json` (permissions 0o600) -2. **Incremental sync** tracked via `sync-tracker.ts` (last sync timestamp in DB) -3. **Retry logic** from `http-utils.ts` (exponential backoff, configurable retries) -4. **Disconnect** removes all documents with matching `sourceType` + source identifier +Each connector type is one entry in `src/connectors/registry.ts`. `libscope connect ` saves a connection in `~/.libscope/connectors/.json` (mode `0600`) and syncs it. The `sync` and `disconnect` operations, the MCP admin `sync` tool, the REST routes and the scheduler all read the saved connection, so credentials are never operation parameters. Sync state is tracked by `sync-tracker.ts`, and HTTP calls retry with `http-utils.ts`. -## Data Flow: Indexing +## Data Flow: Adding content -When you index a document (`libscope add` or `submit-document` MCP tool): +When you run `libscope add`, the MCP tool `submit-document`, `POST /api/v1/documents` or `scope.add()`, the same `add` operation runs: ``` -Input (file/URL/text) +Input (inline content / file / directory / URL / repository URL) + → ingest.ts: detect the kind, fetch or read, crawl (spider) or list files → Parser (markdown.ts / html.ts / pdf.ts / ...) - → Raw text - → Chunker (indexing.ts) — paragraph-aware, heading breadcrumbs, overlap - → Chunks[] - → EmbeddingProvider.embed(chunks) - → Vectors[] - → DB: documents table + chunks table + chunk_embeddings (vector) + chunks_fts (FTS5) + → Chunker (indexing.ts, or a custom Chunker): heading-aware chunks with breadcrumbs + → EmbeddingProvider.embedBatch(chunks) + → DB: documents + chunks + chunk_embeddings (vector) + chunks_fts (FTS5) ``` -The chunker in `src/core/indexing.ts` splits on paragraph boundaries while respecting heading structure. Each chunk carries a breadcrumb of its parent headings so context is preserved across chunk boundaries. - ## Data Flow: Search ``` Query string - → EmbeddingProvider.embed([query]) → query vector + → EmbeddingProvider.embed(query) → query vector ↓ - ┌── Vector search (sqlite-vec ANN cosine similarity) → ranked chunks - │ + ┌── Vector search (sqlite-vec, cosine distance) → ranked chunks └── FTS5 search (BM25, AND then OR fallback) → ranked chunks ↓ Reciprocal Rank Fusion (RRF, k=60) → merged ranked list ↓ -Title boost (1.5× if title contains query words) - ↓ -MMR diversity reranking (optional, diversity param 0–1) +Title boost ↓ -Filter by: library, topic, tags, minRating, maxChunksPerDocument +MMR diversity reranking (optional, diversity 0–1) ↓ -Paginate (limit, offset) +Filters: topic, library, version, sourceType, tags, minRating, maxChunksPerDocument ↓ -Results with scoreExplanation +Paging (limit, offset) → { items, total, limit, offset } ``` See [How Search Works](/guide/how-search-works) for more detail. -## Data Flow: RAG (Ask) +## Data Flow: Ask ``` Question - → search() — retrieve top-K relevant chunks - → Build context string from chunks - → LLM prompt: "Answer based only on this context: ..." - → LLM response (streaming or buffered) - → Return { text, sources: cited chunks } + → search() — top-K chunks + → context prompt from the chunks + → LLM (OpenAI, Anthropic or Ollama) → { mode: "answer", answer, sources, model } + or passthrough (no LLM call) → { mode: "context", contextPrompt, sources } ``` -The LLM integration in `src/core/rag.ts` supports OpenAI, Ollama, Anthropic, and a `passthrough` mode where the application handles the LLM call externally. +`answer()` in `src/core/rag.ts` handles both modes. With `llm.provider` `auto` (the default), the MCP server uses passthrough: the calling assistant writes the answer. ## Database Schema -Key tables (schema version 17): - -| Table | Purpose | -| ------------------ | -------------------------------------------------- | -| `documents` | Document metadata: title, content, library, topic | -| `chunks` | Document chunks: content, chunk_index, document_id | -| `chunk_embeddings` | Vector table (sqlite-vec): embedding per chunk | -| `chunks_fts` | FTS5 virtual table: full-text search index | -| `topics` | Topic hierarchy (id, name, parent_id) | -| `tags` | Tag definitions | -| `document_tags` | Many-to-many document ↔ tag | -| `ratings` | Document and chunk ratings (1–5) | -| `document_versions`| Version history for rollback | -| `document_links` | Typed cross-references between documents | -| `search_log` | Query analytics | -| `document_hits` | Per-document result hit analytics | -| `saved_searches` | Named query persistence | -| `connector_configs`| Connector state (tokens, last sync) | -| `webhooks` | Event webhook configuration | -| `schema_version` | Current migration version | - -## How to Add a New CLI Command - -1. Add a new file in `src/cli/commands/` (or add to an existing file) -2. Export a `registerCommands(program: Command): void` function -3. Import and call it in `src/cli/index.ts` -4. Call the appropriate `src/core/` functions — don't implement logic in the CLI layer - -Example skeleton: +Key tables (schema version 19): + +| Table | Purpose | +| ------------------- | --------------------------------------------------- | +| `documents` | Document metadata and content | +| `chunks` | Document chunks | +| `chunk_embeddings` | Vector table (sqlite-vec): one embedding per chunk | +| `chunks_fts` | FTS5 full-text index | +| `index_meta` | Provider, model and vector size of the vector index | +| `topics` | Topic hierarchy | +| `tags` | Tags | +| `document_tags` | Document ↔ tag | +| `ratings` | Document and chunk ratings (1–5) | +| `document_versions` | Version history for rollback | +| `document_links` | Typed links between documents | +| `packs` | Installed knowledge packs | +| `search_log` | One row per search (method, result count, latency) | +| `search_queries` | Query log with result count and top score | +| `document_hits` | How often each document is returned | +| `saved_searches` | Saved searches | +| `connector_syncs` | Connector sync history | +| `connector_configs` | Not used (connections are saved as files) | +| `webhooks` | Webhooks | +| `schema_version` | Applied migrations | + +## How to Add an Operation + +New functionality starts as an operation. The surfaces then expose it with little or no code. + +1. Write the logic in `src/core/` as a function over the database and provider. +2. Define the operation in the file of its group in `src/core/operations/` with `defineOperation({ name, group, summary, input, annotations, http, handler })`. Use the shared fields from `schemas.ts`, give every field a `.describe()`, and put defaults in the schema. Mark it `longRunning` if it can take long (surfaces then offer a background task). +3. Add it to the group's exported list (for example `topicOperations`), so it is in `OPERATIONS`. +4. Expose it: + - **REST**: automatic when the operation has `http`. The OpenAPI document follows. + - **SDK**: add it to `NAMESPACES` in `src/LibScope.ts` (a test checks that the SDK reaches every operation). + - **CLI**: add a command in the right `src/cli/commands/*.ts` file that calls `run(op, input, format)`. A test checks that every command calls an operation. + - **MCP**: only for tools that an assistant needs. Add it to `registerCoreTools` or `registerAdminTools` in `src/mcp/server.ts` with a formatter in `src/mcp/format.ts`. +5. Run `npm run build && npm run docs:gen` to update the generated references, and commit them. + +Example (a hypothetical `rename-topic` operation over the existing `renameTopic` in `src/core/topics.ts`): ```typescript -// src/cli/commands/my-feature.ts -import { Command } from "commander"; -import { myFeatureCore } from "../../core/my-feature.js"; -import { getDb } from "../../db/connection.js"; - -export function registerCommands(program: Command): void { - program - .command("my-feature ") - .description("Does something useful") - .option("--flag ", "An option") - .action(async (arg, opts) => { - const db = getDb(); - const result = await myFeatureCore(db, arg, opts); - console.log(result); - }); -} +// src/core/operations/topics.ts +export const renameTopicOperation = defineOperation({ + name: "rename-topic", + group: "topics", + summary: "Rename a topic", + input: z.object({ topic: s.topic, name: z.string().min(1).describe("New name") }), + annotations: { idempotent: true }, + http: { method: "PATCH", path: "/topics/:topic" }, + handler: (ctx, input) => renameTopic(ctx.db, resolveTopicId(ctx.db, input.topic), input.name), +}); ``` -## How to Add a New MCP Tool +```typescript +// src/cli/commands/topics.ts, inside register(program) +topics + .command("rename ") + .description("Rename a topic") + .action(async (topic: string, name: string) => { + await run(renameTopicOperation, { topic, name }, (t) => console.log(`✓ Renamed to ${t.name}`)); + }); +``` -1. Add the tool in an existing file under `src/mcp/tools/` (or create a new file) -2. Register the tool via `server.tool(name, description, schema, handler)` -3. Wrap the handler with `withErrorHandling()` from `src/mcp/errors.ts` -4. Export a `registerTools(server, db, provider)` function and call it in `src/mcp/server.ts` +## How to Add a Connector -Example skeleton: +1. Write the connector in `src/connectors/my-connector.ts`: its config type, a sync function and a disconnect function. Use `http-utils.ts` for HTTP calls with retry. +2. Add an entry to `CONNECTORS` in `src/connectors/registry.ts` with the config schema and the secret fields. +3. Add its flags to `libscope connect` in `src/cli/commands/connectors.ts`. -```typescript -import { withErrorHandling } from "../errors.js"; -import { z } from "zod"; - -export function registerTools(server, db, provider) { - server.tool( - "my-tool", - "Does something useful", - { param: z.string().describe("A parameter") }, - withErrorHandling(async ({ param }) => { - const result = await myCore(db, param); - return { content: [{ type: "text", text: JSON.stringify(result) }] }; - }), - ); -} -``` +`sync`, `disconnect`, `connections`, the MCP admin `sync` tool, the REST routes and the scheduler then work for it without more code. -## How to Add a New Connector +## How to Add an Embedding Provider -1. Create `src/connectors/my-connector.ts` -2. Use `http-utils.ts` for HTTP calls with retry -3. Track sync state with `sync-tracker.ts` -4. Store auth config in `~/.libscope/connectors/my-connector.json` (chmod 0o600) -5. Add a CLI command in `src/cli/commands/` under the `connect` subcommand -6. Add an MCP tool in `src/mcp/tools/` +1. Create `src/providers/my-provider.ts` that implements `EmbeddingProvider`. +2. Add it to `createEmbeddingProvider()` in `src/providers/index.ts`. +3. Add the provider name to `EMBEDDING_PROVIDERS` in `src/config-schema.ts`. -## How to Add a New Embedding Provider +## Generated Documentation -1. Create `src/providers/my-provider.ts` implementing `EmbeddingProvider` -2. Export a class with `dimensions: number` and `embed(texts: string[]): Promise` -3. Add it to the provider factory in `src/providers/index.ts` -4. Add the provider name to the config type in `src/config.ts` +The CLI, MCP tool, configuration and REST references contain blocks between `` and `` markers. `scripts/gen-docs.mjs` fills them from the built code: the commander `program`, the tools of `createMcpServer()`, `getConfigKeyTable()` and `API_ROUTES`. Run `npm run build && npm run docs:gen` after changing a command, a tool, a config key or a route. CI runs `npm run docs:check`, which fails when a block is out of date. ## Testing Approach @@ -370,15 +378,15 @@ import { createTestDb } from "../fixtures/test-db.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; import { insertDoc, insertChunk, seedTestDocument } from "../fixtures/helpers.js"; -// Fresh in-memory DB with all migrations applied +// Fresh in-memory database with all migrations applied const db = createTestDb(); -// Deterministic 4-dimensional vectors — no real embedding model required +// Deterministic 4-dimensional vectors: no embedding model needed const provider = new MockEmbeddingProvider(); ``` -- **Unit tests** (`tests/unit/`) — mock all dependencies, test one module at a time, run fast -- **Integration tests** (`tests/integration/`) — real SQLite DB, full indexing → search → rate workflow -- Use `createTestDbWithVec()` when you need the vector table (requires sqlite-vec) +- **Unit tests** (`tests/unit/`): mocked dependencies, one module at a time. Operations are tested in `tests/unit/operations/`. +- **Integration tests** (`tests/integration/`): a real SQLite database and full workflows. +- Use `createTestDbWithVec()` when you need the vector table. -Coverage thresholds enforced in CI: 75% statements, 74% branches, 75% functions, 75% lines. +Coverage thresholds in CI: 75% statements, 74% branches, 75% functions, 75% lines. diff --git a/docs/guide/code-indexing.md b/docs/guide/code-indexing.md index 176adf3d..b92ef6c7 100644 --- a/docs/guide/code-indexing.md +++ b/docs/guide/code-indexing.md @@ -1,6 +1,6 @@ # Code Indexing -LibScope Lite includes a tree-sitter powered code chunker that splits source files at function and class boundaries. This produces semantically meaningful chunks that are far better for embedding than naive line-count splits. +`libscope/lite` includes a tree-sitter code chunker that splits source files at function and class boundaries. These chunks embed much better than splits at a fixed line count. Use `createCodeChunker()` to index code with `LibScope`, or `TreeSitterChunker` to get the chunks with line numbers. ## Why Code-Aware Chunking Matters @@ -14,35 +14,35 @@ The tree-sitter chunker uses the Abstract Syntax Tree (AST) to split at **semant ## Installation -Tree-sitter is an **optional peer dependency**. Install the packages for the languages you need: +Tree-sitter is an **optional peer dependency**. Install the packages for the languages you need, with the versions that libscope supports (the latest grammar releases need a newer tree-sitter, and npm refuses to install them next to libscope): ```bash # Core tree-sitter parser -npm install tree-sitter +npm install tree-sitter@0.21 # Language grammars (install only what you need) -npm install tree-sitter-typescript # TypeScript + TSX -npm install tree-sitter-javascript # JavaScript, JSX, MJS, CJS -npm install tree-sitter-python # Python -npm install tree-sitter-c-sharp # C# -npm install tree-sitter-cpp # C++ (also used for .h/.hpp headers) -npm install tree-sitter-c # C -npm install tree-sitter-go # Go +npm install tree-sitter-typescript@0.21 # TypeScript + TSX +npm install tree-sitter-javascript@0.21 # JavaScript, JSX, MJS, CJS +npm install tree-sitter-python@0.21 # Python +npm install tree-sitter-c-sharp@0.21 # C# +npm install tree-sitter-cpp@0.22 # C++ (also used for .h/.hpp headers) +npm install tree-sitter-c@0.21 # C +npm install tree-sitter-go@0.21 # Go ``` -If tree-sitter is not installed, `TreeSitterChunker.chunk()` throws a `ValidationError` with a clear install message. All other LibScope Lite features work normally without tree-sitter. +If tree-sitter is not installed, `TreeSitterChunker.chunk()` throws a `ValidationError` with the install command, and `createCodeChunker()` falls back to the built-in chunker. Everything else works without tree-sitter. ## Supported Languages -| Language | Aliases | Grammar Package | -|---|---|---| -| TypeScript | `typescript`, `ts`, `tsx` | `tree-sitter-typescript` | +| Language | Aliases | Grammar Package | +| ---------- | --------------------------------------- | ------------------------ | +| TypeScript | `typescript`, `ts`, `tsx` | `tree-sitter-typescript` | | JavaScript | `javascript`, `js`, `jsx`, `mjs`, `cjs` | `tree-sitter-javascript` | -| Python | `python`, `py` | `tree-sitter-python` | -| C# | `csharp`, `cs` | `tree-sitter-c-sharp` | -| C++ | `cpp`, `cc`, `cxx`, `hpp`, `h` | `tree-sitter-cpp` | -| C | `c` | `tree-sitter-c` | -| Go | `go` | `tree-sitter-go` | +| Python | `python`, `py` | `tree-sitter-python` | +| C# | `csharp`, `cs` | `tree-sitter-c-sharp` | +| C++ | `cpp`, `cc`, `cxx`, `hpp`, `h` | `tree-sitter-cpp` | +| C | `c` | `tree-sitter-c` | +| Go | `go` | `tree-sitter-go` | Aliases are case-insensitive: `"TS"`, `"ts"`, `"TypeScript"` all resolve to TypeScript. @@ -78,7 +78,7 @@ export class AuthService extends EventEmitter { private async generateToken(userId: string): Promise { // ... token generation logic - return `tok_${userId}_${Date.now()}`; + return "tok_" + userId + "_" + Date.now(); } } `; @@ -90,21 +90,18 @@ Each chunk in the result: ```ts interface CodeChunk { - content: string; // source text of the chunk - startLine: number; // 1-based start line in the original file - endLine: number; // 1-based end line in the original file - nodeType: string; // tree-sitter node type (see below) + content: string; // source text of the chunk + startLine: number; // 1-based start line in the original file + endLine: number; // 1-based end line in the original file + nodeType: string; // tree-sitter node type (see below) } ``` -For the example above, you'd get chunks like: +For the example above, the class fits in one chunk (under 1500 characters). The import before it is prepended to it as a preamble: ``` -chunk[0]: "import { EventEmitter } from 'events';" - startLine: 2, endLine: 2, nodeType: "preamble" - -chunk[1]: "export class AuthService extends EventEmitter { ... }" - startLine: 4, endLine: 25, nodeType: "class_declaration" +chunk[0]: "import { EventEmitter } from \"events\";\n\nexport class AuthService ... }" + startLine: 2, endLine: 23, nodeType: "export_statement" ``` ## Node Types @@ -112,6 +109,7 @@ chunk[1]: "export class AuthService extends EventEmitter { ... }" The chunker extracts these node types per language: **TypeScript / TSX:** + - `function_declaration` — `function foo() {}` - `class_declaration` — `class Foo {}` - `method_definition` — methods inside a class @@ -122,14 +120,17 @@ The chunker extracts these node types per language: - `enum_declaration` — TypeScript enums **JavaScript / JSX:** + - `function_declaration`, `class_declaration`, `method_definition`, `export_statement`, `lexical_declaration` **Python:** + - `function_definition` — `def foo():` - `class_definition` — `class Foo:` - `decorated_definition` — `@decorator\ndef foo():` **C#:** + - `method_declaration` — `public void Foo() {}` - `class_declaration` — `class Foo {}` - `interface_declaration` — `interface IFoo {}` @@ -138,16 +139,19 @@ The chunker extracts these node types per language: - `constructor_declaration` — `public Foo() {}` **C++:** + - `function_definition` — `void foo() {}` - `class_specifier` — `class Foo {}` - `struct_specifier` — `struct Point {}` - `namespace_definition` — `namespace MyNS {}` **C:** + - `function_definition` — `void foo() {}` - `struct_specifier` — `struct Point {}` **Go:** + - `function_declaration` — `func Foo() {}` - `method_declaration` — `func (r *Receiver) Foo() {}` - `type_declaration` — `type Foo struct {}` / `type Bar interface {}` @@ -184,58 +188,35 @@ When a class is split, each method becomes its own chunk. If a single method is If the source has no declaration nodes (e.g., a config file, a `.d.ts` with only type exports), the entire source is returned as a single chunk with `nodeType: "module"`. -## Integrating with LibScope Lite +## Indexing code with LibScope -The typical pattern for indexing a codebase: +`createCodeChunker()` returns a `Chunker` for `createLite()` or `LibScope.create()`. `add()` then splits each document with tree-sitter. The language comes from the extension of the file path, URL or title (or the `language` option). Documents in other languages use the built-in chunker. ```ts -import { LibScopeLite, TreeSitterChunker } from "libscope/lite"; +import { createLite, createCodeChunker } from "libscope/lite"; import { readdir, readFile } from "node:fs/promises"; -import { join, extname } from "node:path"; +import { join } from "node:path"; -const chunker = new TreeSitterChunker(); -const lite = new LibScopeLite({ dbPath: "./my-project.db" }); - -async function indexDirectory(dir: string): Promise { - const entries = await readdir(dir, { recursive: true, withFileTypes: true }); +const scope = createLite({ dbPath: "./my-project.db", chunker: createCodeChunker() }); - const tasks = entries +const entries = await readdir("./src", { recursive: true, withFileTypes: true }); +await Promise.all( + entries .filter((e) => e.isFile()) - .map(async (entry) => { - const filePath = join(entry.parentPath, entry.name); - const ext = extname(entry.name).slice(1); // "ts", "py", etc. - const source = await readFile(filePath, "utf8"); - - if (chunker.supports(ext)) { - // Code-aware chunking - const chunks = await chunker.chunk(source, ext); - return chunks.map((c) => ({ - title: `${filePath}:${c.startLine}-${c.endLine}`, - content: c.content, - url: filePath, - library: "src", - })); - } - - // Plain text fallback for unsupported files - return [{ title: filePath, content: source, url: filePath, library: "src" }]; - }); - - const docGroups = await Promise.all(tasks); - await lite.indexBatch(docGroups.flat(), { concurrency: 4 }); -} + .map(async (e) => { + const path = join(e.parentPath, e.name); + await scope.add({ title: path, content: await readFile(path, "utf8"), library: "src" }); + }), +); -await indexDirectory("./src"); -console.log("Indexed. Searching..."); +const { items } = await scope.search("authentication token generation"); +for (const r of items) console.log(`${r.title} (score: ${r.score.toFixed(3)})`); -const results = await lite.search("authentication token generation"); -for (const r of results) { - console.log(`${r.title} (score: ${r.score.toFixed(3)})`); -} - -lite.close(); +scope.close(); ``` +`scope.add(path)` does not accept source-code extensions such as `.ts` or `.py` (only document formats). Read the file and pass its `content` with the path as `title`, as above. + ## Caching `TreeSitterChunker` lazily initializes the tree-sitter parser and grammar modules on first use and caches them for the lifetime of the instance. Create one `TreeSitterChunker` instance and reuse it rather than creating a new one per file: @@ -263,15 +244,16 @@ try { const chunks = await chunker.chunk(source, "rust"); } catch (err) { if (err instanceof ValidationError) { - // "Unsupported language for code chunking: 'rust'" - // "Code chunking requires the 'tree-sitter' package. Install it with: ..." + // Unsupported language for code chunking: "rust" + // Code chunking requires the "tree-sitter" package. Install it with: ... console.warn(err.message); } } ``` Two error conditions: -1. **Unsupported language** — throws immediately with the list of supported aliases + +1. **Unsupported language** — throws immediately (the aliases are in [Supported Languages](#supported-languages)) 2. **tree-sitter not installed** — throws with the exact `npm install` command Both are `ValidationError` from LibScope's error hierarchy. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 7a47ed87..54a2fae3 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1,45 +1,65 @@ # Configuration -LibScope uses a 3-tier config system. Higher tiers override lower ones: +LibScope reads settings from these layers. Higher layers override lower ones: **Environment variables** > **Project `.libscope.json`** > **User `~/.libscope/config.json`** > **Defaults** +API keys are the exception: LibScope reads them only from environment variables and `~/.libscope/secrets.json`. See [API keys](#api-keys). + +Every setting has an environment variable named `LIBSCOPE_
_`, for example `embedding.model` → `LIBSCOPE_EMBEDDING_MODEL`. The [configuration reference](../reference/configuration.md) lists all keys. Keys renamed in 2.0 are listed in the [migration guide](../migration-v2.md#configuration). + ## Config File You can set options via the CLI or by editing the config file directly. ```bash -# Set a value +# Set a value (any key from the configuration reference) libscope config set embedding.provider ollama -# View current config +# Read or remove one value +libscope config get embedding.provider +libscope config unset embedding.provider + +# Show where the user config file is +libscope config path + +# View current config (API keys masked) libscope config show ``` +`config set` validates the key and value, and changes only that key. Other content of the file, such as `registries`, is kept. API keys are written to `~/.libscope/secrets.json`. All other keys are written to `~/.libscope/config.json`. + Example `~/.libscope/config.json`: ```json { "embedding": { - "provider": "local", - "ollamaUrl": "http://localhost:11434", - "ollamaModel": "nomic-embed-text", - "openaiModel": "text-embedding-3-small" + "provider": "ollama", + "url": "http://localhost:11434", + "model": "nomic-embed-text" }, "llm": { "provider": "openai", "model": "gpt-4o-mini" }, - "database": { - "path": "~/.libscope/libscope.db" - }, "logging": { "level": "info" } } ``` -You can also create a `.libscope.json` in your project root for per-project settings. This is useful if different projects use different embedding providers or databases. +You can also create a `.libscope.json` in your project root for per-project settings. This is useful if different projects use different embedding providers or databases. In `.libscope.json`, set `"workspace": ""` to use a workspace, or `database.path` to use a specific file. + +## Database Location + +By default the database is the active workspace's file: `~/.libscope/workspaces//libscope.db` (workspace `default` unless you choose another). The CLI, MCP server, REST API, and `LibScope.create()` use the same rule: + +1. `database.path`, if set in `.libscope.json` or `~/.libscope/config.json` (or `dbPath` passed to `LibScope.create()`). A leading `~` is expanded. +2. Otherwise the workspace: `--workspace` / `LibScope.create({ workspace })`, then `LIBSCOPE_WORKSPACE`, then `"workspace"` in `.libscope.json`, then `libscope workspace use`, then `default`. + +`libscope doctor` shows the workspace and the database file in use. + +Earlier versions of the SDK stored data in `~/.libscope/libscope.db`. If that file exists and the workspace database does not, LibScope prints a warning once. To keep using the old file, run `libscope config set database.path ~/.libscope/libscope.db`. LibScope does not move or delete it. ## Embedding Providers @@ -49,66 +69,91 @@ Embeddings turn text into vectors for semantic search. LibScope supports three p | -------- | ------- | ---------------------- | --------------------------------------------- | | `local` | ✅ | Nothing | all-MiniLM-L6-v2, ~80MB download on first use | | `ollama` | | Ollama running locally | Uses nomic-embed-text by default | -| `openai` | | API key | Uses text-embedding-3-small | +| `openai` | | API key | Uses text-embedding-3-small by default | The local provider works out of the box — no API keys, no external services. It runs the model in-process using `@xenova/transformers`. ```bash # Switch to Ollama libscope config set embedding.provider ollama +libscope config set embedding.url http://gpu-box:11434 # optional, default http://localhost:11434 +libscope config set embedding.model mxbai-embed-large # optional # Or OpenAI libscope config set embedding.provider openai +libscope config set openai.apiKey sk-... # or export OPENAI_API_KEY +``` + +`embedding.model` and `embedding.url` apply to the selected provider. The local provider always uses all-MiniLM-L6-v2 and ignores `embedding.model`. + +LibScope records the provider, model, and vector size that built the vector index. If you change the provider, the model, or `embedding.dimensions` after you index documents, LibScope stops with an error that names the old and the new model. To rebuild the vector index with the new model, run: + +```bash +libscope admin reindex --rebuild ``` -If you switch providers after indexing documents, run `libscope reindex` to re-embed existing chunks with the new model. +LibScope knows the vector size of common models (for example `nomic-embed-text`, `mxbai-embed-large`, `all-minilm`, `text-embedding-3-small`, and `text-embedding-3-large`). For a different Ollama or OpenAI model, set `embedding.dimensions` to the vector size of the model. If you do not set it, `libscope admin reindex --rebuild` embeds one probe text to find the size. For `text-embedding-3-*` models, `embedding.dimensions` also asks OpenAI for shorter vectors. ## LLM Configuration -The `ask` command and the `ask-question` MCP tool use an LLM to synthesize answers from search results (RAG). This requires a separate LLM provider: +The `ask` command, the `ask` MCP tool and `POST /api/v1/ask` use an LLM to write answers from search results (RAG). + +`llm.provider` is `auto` by default. LibScope selects the LLM when `ask` runs, without network calls: + +1. Under the MCP server: `passthrough`. LibScope returns the retrieved context, and the calling assistant writes the answer. +2. Otherwise, `openai` if an OpenAI key is set. +3. Otherwise, `anthropic` if an Anthropic key is set. +4. Otherwise, `ollama` if `llm.url` is set or `embedding.provider` is `ollama`. +5. Otherwise there is no LLM, and `ask` fails with a message that tells you what to set. + +To choose the provider yourself: ```bash # Via config -libscope config set llm.provider openai +libscope config set llm.provider openai # auto | openai | anthropic | ollama | passthrough +libscope config set llm.model gpt-4o-mini # optional # Via environment variables export LIBSCOPE_LLM_PROVIDER=openai export LIBSCOPE_LLM_MODEL=gpt-4o-mini ``` -Supported providers: `openai`, `ollama`, `anthropic`, `passthrough`. +For Ollama, `llm.url` sets the server. If it is not set, LibScope uses `embedding.url`, then `http://localhost:11434`. -The `anthropic` provider uses Anthropic's Claude models. Set the API key via config or environment variable: +The `anthropic` provider uses Anthropic's Claude models: ```bash -export LIBSCOPE_LLM_PROVIDER=anthropic -export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... +export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... # or: libscope config set anthropic.apiKey sk-ant-... ``` -Or in your config file, set `llm.provider` to `"anthropic"` and `llm.anthropicApiKey` to your key. You can optionally set `llm.model` to choose a specific Claude model. +With `llm.provider` set to `auto`, this key alone selects Anthropic (when no OpenAI key is set). You can set `llm.model` to choose a specific Claude model. + +The `passthrough` provider does not call an LLM. `ask` then returns the retrieved context, and the caller (for example the assistant that called the MCP `ask` tool) writes the answer. `libscope doctor` shows which LLM `ask` will use. -The `passthrough` provider is for advanced integrations where you supply your own LLM responses externally. When set, the `ask` command emits an event stream that your application handles rather than calling an LLM directly. +## API keys + +LibScope reads each API key in this order: + +1. `LIBSCOPE_OPENAI_API_KEY` / `LIBSCOPE_ANTHROPIC_API_KEY` +2. `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` +3. `~/.libscope/secrets.json`, written by `libscope config set openai.apiKey ` or `libscope config set anthropic.apiKey ` (mode `0600`) + +The OpenAI key is used for OpenAI embeddings and the OpenAI LLM. LibScope ignores API keys in `config.json` and `.libscope.json`, and logs a warning. `libscope config show` and `libscope config get` mask keys. ## Environment Variables -| Variable | Description | Default | -| ---------------------------------- | -------------------------------------------------- | ------------------------ | -| `LIBSCOPE_EMBEDDING_PROVIDER` | Embedding provider (`local` / `ollama` / `openai`) | `local` | -| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key | — | -| `LIBSCOPE_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` | -| `LIBSCOPE_LLM_PROVIDER` | LLM provider for RAG (`openai` / `ollama` / `anthropic`) | — | -| `LIBSCOPE_LLM_MODEL` | LLM model override | — | -| `LIBSCOPE_ANTHROPIC_API_KEY` | Anthropic API key (for Claude models) | — | -| `LIBSCOPE_ALLOW_PRIVATE_URLS` | Allow fetching from private/internal IPs | `false` | -| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | Accept self-signed TLS certificates | `false` | -| `ONENOTE_CLIENT_ID` | Microsoft app registration client ID | — | -| `ONENOTE_TENANT_ID` | Microsoft tenant ID | `common` | -| `NOTION_TOKEN` | Notion integration token | — | -| `CONFLUENCE_URL` | Confluence base URL | — | -| `CONFLUENCE_EMAIL` | Confluence user email | — | -| `CONFLUENCE_TOKEN` | Confluence API token | — | - -Environment variables always take precedence over config files. +Every config key has an environment variable `LIBSCOPE_
_`, for example `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_LLM_MODEL` or `LIBSCOPE_MCP_TOOLSETS`. The API keys also accept `OPENAI_API_KEY` and `ANTHROPIC_API_KEY`. The [configuration reference](../reference/configuration.md#all-config-keys) lists every key with its variable and default. + +These variables are not config keys: + +| Variable | Description | +| --------------------- | ---------------------------------------------------------- | +| `LIBSCOPE_WORKSPACE` | Active workspace for this shell (default `default`) | +| `LIBSCOPE_API_KEY` | REST API key; requests must send `Authorization: Bearer …` | +| `LIBSCOPE_SECRET_KEY` | Encrypts stored webhook secrets | +| `LIBSCOPE_VERBOSE` | `1` prints structured JSON logs (stderr) in the CLI | + +Environment variables take precedence over config files. LibScope does not load `.env` files; export the variables in your shell or MCP client config. Connector credentials are not read from environment variables: give them to `libscope connect` (see [Connectors](./connectors.md)). ## Workspaces diff --git a/docs/guide/connectors.md b/docs/guide/connectors.md index 53c0e388..a61985b3 100644 --- a/docs/guide/connectors.md +++ b/docs/guide/connectors.md @@ -1,141 +1,190 @@ # Connectors -Connectors pull documents from external tools into your LibScope knowledge base. Each connector handles authentication, pagination, and incremental sync so you don't have to think about it. +Connectors pull documents from external tools into your LibScope knowledge base: Notion, Slack, Confluence, Obsidian, OneNote and documentation sites. Each connector handles authentication, pagination and incremental sync. + +## Connections + +`libscope connect ` saves a **connection** and syncs it. A connection is the connector's settings, including credentials, in `~/.libscope/connectors/.json` (file mode `0600`). The default name is the connector type. Use `--name` to keep more than one connection of the same type: + +```bash +libscope connect slack --token xoxb-work-token --name work-slack +``` + +| Command | What it does | +| -------------------------------------------- | --------------------------------------------------------------- | +| `libscope connect [source] [options]` | Create or update a connection, then sync it (`--no-sync` skips) | +| `libscope sync ` | Sync one connection with its saved settings | +| `libscope sync --all` | Sync every connection | +| `libscope connections` | List connections with their schedule, last sync and last run | +| `libscope disconnect ` | Delete the connection's documents and its saved settings | + +Running `connect` again for a saved connection changes only the settings you give and keeps the rest. `disconnect --keep-documents` deletes only the saved settings. The [CLI reference](/reference/cli#connectors) lists every option. + +A connection holds these settings: + +| Type | Saved settings | +| ------------ | ---------------------------------------------------------------------------------------------------- | +| `notion` | `token`, `excludePages` | +| `slack` | `token`, `channels`, `excludeChannels`, `threadMode` | +| `confluence` | `baseUrl`, `type` (cloud or server), `email`, `token`, `spaces`, `excludeSpaces` | +| `obsidian` | `vaultPath`, `topicMapping`, `excludePatterns` | +| `onenote` | `clientId`, `tenantId`, `accessToken`, `refreshToken`, `tokenExpiry`, `notebooks`, `excludeSections` | +| `docs` | `url`, `type` (site generator), `library`, `version`, `maxPages`, `maxDepth`, `pathPrefix` | + +Each file also stores `connectorType`, the `schedule` (if set) and `lastSync`. After each successful sync, `lastSync` is set to the time the sync started. Notion, Slack and OneNote use it to fetch only what changed since then. Obsidian re-indexes only files whose modification time changed. + +### Schedules + +`--schedule ` syncs a connection on a cron schedule. Schedules run while the REST API server runs (`libscope serve api`). For a saved connection, give only the schedule (a new connection also needs its other settings, such as `--token`): + +```bash +libscope connect notion --schedule "0 */6 * * *" --no-sync # every 6 hours +libscope connections # shows the schedule +libscope connect notion --schedule off --no-sync # remove the schedule +``` + +Each run, scheduled or manual, records its status. `libscope connections` shows the last run. ## Obsidian -Sync an entire Obsidian vault. Parses frontmatter, wikilinks, embeds, and tags. Folder structure maps to topics. +Sync an Obsidian vault. LibScope parses frontmatter, wikilinks, embeds and tags. The folder structure maps to topics. ```bash -# Initial sync +# First sync libscope connect obsidian /path/to/vault -# Map topics from frontmatter instead of folder structure +# Map topics from frontmatter instead of folders libscope connect obsidian /path/to/vault --topic-mapping frontmatter -# Incremental re-sync (only changed files) -libscope connect obsidian /path/to/vault --sync +# Skip folders (comma-separated globs) +libscope connect obsidian /path/to/vault --exclude "templates/*,daily/*" -# Exclude folders -libscope connect obsidian /path/to/vault --exclude "templates/*" "daily/*" +# Sync again (only changed files are re-indexed) +libscope sync obsidian -# Remove vault data from LibScope -libscope disconnect obsidian /path/to/vault +# Remove the vault's documents and the connection +libscope disconnect obsidian ``` ## Notion -Sync pages and databases from your Notion workspace. Requires a [Notion integration token](https://www.notion.so/my-integrations). +Sync pages and databases from your Notion workspace. You need a [Notion integration token](https://www.notion.so/my-integrations). ```bash -# Sync with integration token +# Sync with an integration token libscope connect notion --token secret_abc123 -# Exclude specific pages or databases -libscope connect notion --token $NOTION_TOKEN --exclude page-id-1 db-id-2 +# Skip pages or databases (comma-separated IDs) +libscope connect notion --exclude page-id-1,db-id-2 -# Re-sync (uses stored token) -libscope connect notion --sync +# Sync again (pages edited since the last sync) +libscope sync notion -# Disconnect libscope disconnect notion ``` ## Confluence -Sync Confluence Cloud spaces and pages. Requires an [API token](https://id.atlassian.com/manage-profile/security/api-tokens). +Sync Confluence spaces and pages. For Confluence Cloud you need an [API token](https://id.atlassian.com/manage-profile/security/api-tokens). The base URL is the `[source]` argument. ```bash -# Sync all spaces -libscope connect confluence \ - --url https://acme.atlassian.net \ +# All spaces (Cloud) +libscope connect confluence https://acme.atlassian.net \ --email user@acme.com \ - --token $CONFLUENCE_TOKEN + --token "$CONFLUENCE_API_TOKEN" -# Sync specific spaces -libscope connect confluence \ - --url https://acme.atlassian.net \ +# Some spaces, skip one +libscope connect confluence https://acme.atlassian.net \ --email user@acme.com \ - --token $CONFLUENCE_TOKEN \ + --token "$CONFLUENCE_API_TOKEN" \ --spaces ENG,DEVOPS \ - --exclude-spaces ARCHIVE + --exclude ARCHIVE + +# Server or Data Center with a personal access token +libscope connect confluence https://wiki.internal.example.com --server --token "$CONFLUENCE_PAT" -# Disconnect +libscope sync confluence libscope disconnect confluence ``` +The examples read the token from a shell variable to keep it out of your shell history. LibScope does not read connector credentials from environment variables. + ## Slack -Index Slack channel messages and threads. Requires a [Slack bot token](https://api.slack.com/authentication/token-types#bot) with appropriate scopes. +Index Slack channel messages and threads. You need a [Slack bot token](https://api.slack.com/authentication/token-types#bot) with the read scopes for the channels. ```bash -# Sync all channels +# All channels libscope connect slack --token xoxb-your-bot-token -# Sync specific channels -libscope connect slack \ - --token xoxb-... \ - --channels general,engineering \ - --thread-mode aggregate +# Some channels +libscope connect slack --token xoxb-... --channels general,engineering --thread-mode aggregate # Thread modes: -# aggregate — combines thread replies into one document (default) -# separate — one document per reply +# aggregate: one document per thread (default) +# separate: one document per reply -# Re-sync -libscope connect slack --sync +# Sync again (messages since the last sync) +libscope sync slack -# Disconnect libscope disconnect slack ``` ## OneNote -Sync OneNote notebooks via the Microsoft Graph API. Uses device code authentication — you'll be prompted to open a browser and log in. +Sync OneNote notebooks through the Microsoft Graph API. LibScope signs in with a device code: it prints a URL and a code, and you sign in in a browser. ```bash -# Set your app registration client ID -export ONENOTE_CLIENT_ID=your-client-id - -# Authenticate and sync -libscope connect onenote +# Sign in and sync (needs an Azure app registration client ID) +libscope connect onenote --client-id your-client-id -# Sync a specific notebook +# One notebook only libscope connect onenote --notebook "Work Notes" -# Re-sync with token refresh -libscope connect onenote --sync +# Sync again (the access token is refreshed when it has expired) +libscope sync onenote + +# Use an access token instead of signing in (it cannot be refreshed) +libscope connect onenote --token "$GRAPH_ACCESS_TOKEN" -# Disconnect libscope disconnect onenote ``` -You'll need an Azure AD app registration with `Notes.Read` permission. See [Microsoft's guide](https://learn.microsoft.com/en-us/graph/auth-register-app-v2) for setup. +You need an Azure AD app registration with the `Notes.Read` permission. See [Microsoft's guide](https://learn.microsoft.com/en-us/graph/auth-register-app-v2). `--tenant-id` defaults to `common`. + +## Documentation sites + +Crawl a documentation site (Sphinx, VitePress, Doxygen or a generic site). The site type is detected unless you set `--site-type`. + +```bash +libscope connect docs https://docs.example.com --library example --lib-version 2.0 +libscope connect docs https://docs.example.com --name example-guide --path-prefix /guide --max-pages 200 +libscope sync example-guide +``` + +The defaults are 500 pages and a link depth of 10. -## GitHub / GitLab +## GitHub and GitLab repositories -Index documentation from any GitHub or GitLab repository. +A repository is not a connector. Add it with `libscope add`: ```bash -# Public repo -libscope add-repo https://github.com/org/repo +# Public repository +libscope add https://github.com/org/repo -# Private repo with token, specific branch and path -libscope add-repo https://github.com/org/private-repo \ - --token $GITHUB_TOKEN \ +# Private repository, one branch, some folders and file types +libscope add https://github.com/org/private-repo \ + --token "$GITHUB_TOKEN" \ --branch develop \ - --path docs/ \ + --path docs \ --extensions .md,.mdx,.rst ``` -## MCP Usage +See [Code Indexing](/guide/code-indexing) for source code. -All connectors are also available as MCP tools, so your AI assistant can trigger syncs directly: +## MCP and REST -- `sync-obsidian-vault` -- `sync-notion` -- `sync-confluence` -- `sync-slack` -- `sync-onenote` +Credentials are never tool or request parameters. Save the connection with `libscope connect` first. Then: -See the [MCP Tools Reference](/reference/mcp-tools) for parameter details. +- **MCP**: the `sync` tool in the admin toolset (`LIBSCOPE_MCP_TOOLSETS=admin`) syncs one connection (`{"name": "notion"}`) or all of them (`{"all": true}`). With `async: true` it returns a task ID; `task {"action": "cancel", "taskId": "..."}` stops the sync. See the [MCP Tools Reference](/reference/mcp-tools#sync). +- **REST**: `POST /api/v1/sync` with `{ "name": "notion" }` or `{ "all": true }` starts a task. `GET /api/v1/connections` lists the connections (secrets masked), and `DELETE /api/v1/connections/:name` disconnects one. See the [REST API reference](/reference/rest-api). diff --git a/docs/guide/dashboard.md b/docs/guide/dashboard.md index 7ffa6fe5..179de667 100644 --- a/docs/guide/dashboard.md +++ b/docs/guide/dashboard.md @@ -5,20 +5,20 @@ LibScope includes a local web dashboard for browsing, searching, and managing yo ## Starting the Dashboard ```bash -libscope serve --dashboard +libscope serve dashboard ``` This starts an HTTP server at `http://localhost:3377` by default. ```bash # Use a custom port -libscope serve --dashboard --port 8080 +libscope serve dashboard --port 8080 # Bind to all interfaces (for LAN access) -libscope serve --dashboard --host 0.0.0.0 --port 3377 +libscope serve dashboard --host 0.0.0.0 --port 3377 ``` -The dashboard is fully self-contained — no external CDN dependencies. +The dashboard page needs no external files. The knowledge graph page (`/graph`) loads d3 from `d3js.org`. ## Features @@ -51,7 +51,7 @@ The **Topics** panel shows your topic hierarchy. Clicking a topic filters the do Navigate to `/graph` (e.g. `http://localhost:3377/graph`) to view an interactive visualization of your knowledge base: - **Nodes** represent documents -- **Edges** represent cross-reference links (`link-documents`, `libscope link`) +- **Edges** represent cross-reference links (`libscope docs link`, the `link-documents` MCP tool, `POST /api/v1/documents/:documentId/links`) - **Clusters** are automatically detected and color-coded by topic - Hover a node to see the document title; click to open it in the document browser @@ -61,33 +61,48 @@ The graph is useful for discovering how your documents relate to each other and Click the sun/moon icon in the top-right corner to toggle between light and dark mode. The preference is saved in `localStorage`. +## Data API + +The dashboard pages read their data from a small JSON API on the same server. Each URL runs one LibScope operation, the same operation as the CLI, MCP server and REST API: + +| URL | Operation | +| --------------------------------------------- | --------------------------------------------- | +| `GET /api/stats` | `overview` (document, topic and chunk counts) | +| `GET /api/topics` | `list-topics` | +| `GET /api/documents?limit&offset&topic` | `list-documents` | +| `GET /api/documents/:id` | `get-document` | +| `DELETE /api/documents/:id` | `delete-document` | +| `GET /api/search?q&limit&topic` | `search` | +| `GET /api/graph?threshold&maxNodes&topic&tag` | `graph` | + +These URLs are for the dashboard pages. For your own scripts, use the [REST API](../reference/rest-api.md) (`libscope serve api`), which has every operation, API-key authentication and an OpenAPI document. + ## Rate Limiting -The dashboard server applies per-IP rate limiting to prevent abuse when exposed on a network. The defaults are generous for local use; if you hit limits, restart with `--host localhost` to restrict access to your machine. +The dashboard server allows 120 requests per minute from each IP address. A request over the limit gets `429`. ## Running Alongside the MCP Server -The dashboard and MCP server run on different ports and can be started independently: +The dashboard, the REST API and the MCP server are separate servers. You can start them independently: ```bash # Dashboard in one terminal -libscope serve --dashboard --port 3377 +libscope serve dashboard --port 3377 -# MCP server in another (or configured in your AI client) +# REST API in another +libscope serve api --port 3378 + +# MCP server (or configure it in your AI client) libscope serve ``` -Or run the REST API and use it as the dashboard's data source: - -```bash -libscope serve --api --port 3378 -libscope serve --dashboard --port 3377 -``` +All of them use the same workspace database. ## Security Considerations -The dashboard is designed for **local use only**. If you expose it on a network: +The dashboard is for **local use**. It has no authentication. Anyone who can connect to its port can read and delete documents. -- Use `--host` carefully — binding to `0.0.0.0` exposes it to all network interfaces -- Consider putting it behind a reverse proxy (nginx, Caddy) with authentication -- The REST API (`--api`) does require an `X-API-Key` header for write operations; the dashboard uses the same key automatically when running on the same host +- The server listens on `localhost` by default. If you use `--host 0.0.0.0`, every network interface can reach it. +- The server sends no `Access-Control-Allow-Origin` header, so pages from other origins cannot read its responses. +- The server rejects write requests (`DELETE`) from pages on other origins with `403`. +- To expose the dashboard on a network, put it behind a reverse proxy (nginx, Caddy) that adds authentication. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index d332585e..9b02d4b4 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -12,15 +12,13 @@ npm install -g libscope Requires Node.js 20 or later. -## Initialize - -Create the database: +## Check the Setup ```bash -libscope init +libscope doctor ``` -This sets up a SQLite database at `~/.libscope/libscope.db` with vector search and full-text indexing. On first run, it also downloads the local embedding model (~80MB). +This shows the config files, the active workspace and its database (`~/.libscope/workspaces/default/libscope.db`), the embedding model, and which LLM `ask` will use. Each warning comes with the command that fixes it. You do not have to create anything first: the database is created when you add the first document (or run `libscope doctor --fix`). The first command that embeds text downloads the local embedding model (~80MB, once) and prints one line while it does. ## Index Some Documents @@ -31,11 +29,16 @@ libscope add ./docs/getting-started.md --library my-lib # A URL (fetches and converts to markdown automatically) libscope add https://docs.example.com/guide -# An entire directory -libscope import ./docs/ --library my-lib --extensions .md,.mdx +# An entire directory (keep it in sync with --watch) +libscope add ./docs/ --library my-lib --include "*.md,*.mdx" + +# A GitHub or GitLab repository +libscope add https://github.com/org/repo --path docs ``` -LibScope supports **Markdown** (`.md`, `.mdx`) and **plain text** natively. Additional formats — **PDF** (`.pdf`), **Word** (`.docx`), **EPUB** (`.epub`), and **PowerPoint** (`.pptx`) — are available via optional dependencies that install automatically. See the [README](/) for the full format table. +Each added document is printed with its ID. Add `--dry-run` to see what would be added. + +LibScope reads **Markdown** (`.md`, `.markdown`, `.mdx`), **plain text** (`.txt`), **HTML** (`.html`, `.htm`), **EPUB** (`.epub`), **PowerPoint** (`.pptx`), **CSV**, **JSON** and **YAML**. **PDF** (`.pdf`) and **Word** (`.docx`) need the optional dependencies `pdf-parse` and `mammoth`, which npm installs by default. Each document gets chunked by heading, embedded into vectors, and stored in the database. @@ -46,20 +49,27 @@ Each document gets chunked by heading, embedded into vectors, and stored in the libscope search "how to authenticate" # Filtered by library and topic -libscope search "API rate limiting" --library my-lib --topic security --limit 10 +libscope search "API rate limiting" --library my-lib --topic security -n 10 + +# Interactive: type queries one after another +libscope search + +# JSON for scripts +libscope search "API rate limiting" --json ``` -Results are ranked by vector similarity, with optional FTS5 boosting for keyword matches. +Results combine vector similarity and keyword (FTS5) matches; see [How Search Works](/guide/how-search-works). Each result shows its document ID and chunk ID; `libscope docs show ` prints the whole document. ## Ask Questions If you have an LLM provider configured (OpenAI, Ollama, or Anthropic), you can ask questions and get synthesized answers with source citations: ```bash +libscope config set openai.apiKey sk-... # or anthropic.apiKey; stored in ~/.libscope/secrets.json libscope ask "What is the recommended auth flow?" ``` -See [Configuration](/guide/configuration) for LLM setup. +Without an LLM, `ask` stops with a hint and `libscope doctor` shows what to set. See [Configuration](/guide/configuration) for LLM setup. ## Start the MCP Server @@ -74,7 +84,7 @@ This starts a stdio-based MCP server that any compatible AI assistant can connec Run the local web dashboard to browse, search, and manage your knowledge base in a browser: ```bash -libscope serve --dashboard +libscope serve dashboard # opens at http://localhost:3377 ``` @@ -86,22 +96,22 @@ Once you have content indexed you can enrich it: ```bash # Tag documents -libscope tag add typescript,api,v2 +libscope docs tag typescript,api,v2 # Group into topics libscope topics create "backend" libscope topics create "auth" --parent backend # Save frequent searches -libscope search "auth best practices" --save "Auth Docs" +libscope searches save "Auth Docs" "auth best practices" --topic auth libscope searches run "Auth Docs" # Cross-reference documents -libscope link --type prerequisite +libscope docs link --type prerequisite # Bulk operations -libscope bulk retag --library react --add-tags deprecated --dry-run -libscope bulk move --library react --to new-topic-id +libscope bulk retag --library react --add deprecated --dry-run +libscope bulk move --library react --to frontend ``` ## REST API @@ -109,10 +119,10 @@ libscope bulk move --library react --to new-topic-id For programmatic access, start the REST API instead of the MCP server: ```bash -libscope serve --api --port 3378 +libscope serve api --port 3378 ``` -The OpenAPI 3.0 spec is served at `GET /openapi.json`. See [REST API Reference](/reference/rest-api) for full documentation. +The OpenAPI 3.1 document is served at `GET /openapi.json`. See [REST API Reference](/reference/rest-api) for full documentation. ## What's Next diff --git a/docs/guide/how-search-works.md b/docs/guide/how-search-works.md index 4f2e9061..afa7ba05 100644 --- a/docs/guide/how-search-works.md +++ b/docs/guide/how-search-works.md @@ -5,24 +5,29 @@ LibScope uses a hybrid search strategy combining vector (semantic) search with f ## Search Pipeline ### 1. Query Embedding -Your search query is converted to a vector embedding using the configured embedding provider (local model or OpenAI). This captures the semantic meaning of your query. + +Your search query is converted to a vector embedding using the configured embedding provider (local model, Ollama or OpenAI). This captures the semantic meaning of your query. ### 2. Vector Search (ANN) + The query vector is matched against all indexed chunk embeddings using approximate nearest-neighbour (ANN) search via `sqlite-vec`. Results are ranked by cosine similarity — chunks semantically related to your query rank highest, even if they use different words. ### 3. Full-Text Search (FTS5) + Simultaneously, SQLite's FTS5 (BM25 ranking) searches for chunks containing your query terms. LibScope first tries AND logic (all terms must match) for precision, then falls back to OR logic if no results are found. ### 4. Hybrid Fusion (RRF) + Vector and FTS5 results are merged using **Reciprocal Rank Fusion (RRF)** — a technique that combines ranked lists without needing calibrated scores: ``` RRF_score(chunk) = Σ 1 / (k + rank_in_list) ``` -where `k = 60` (standard constant). Chunks that rank well in *both* vector and FTS5 lists get the highest fused scores. +where `k = 60` (standard constant). Chunks that rank well in _both_ vector and FTS5 lists get the highest fused scores. ### 5. Title Boost + Chunks whose document title contains any query word receive a 1.5× score multiplier, lifting exact-title matches to the top. ### 6. MMR Diversity Reranking @@ -35,21 +40,24 @@ When you set the `diversity` option (0–1), results are reranked using **Maxima MMR is applied after title boost and score sorting. It's useful when you want to cover different aspects of a topic rather than getting multiple chunks from the same document. +`diversity` is a parameter of the `search` MCP tool, `GET /api/v1/search` and `scope.search()`. The CLI has no flag for it. + ```bash -libscope search "authentication" --diversity 0.5 +curl "http://localhost:3378/api/v1/search?query=authentication&diversity=0.5" ``` ### 7. Pagination & Deduplication -Results are optionally deduplicated by document (`maxChunksPerDocument`) and paginated. Use `offset` and `limit` in your search options for pagination. + +Results are optionally deduplicated by document (`maxChunksPerDocument`) and paginated. Use `offset` and `limit` for pagination (CLI: `--offset` and `-n, --limit`). ## Search Methods -| Method | When Used | Best For | -|--------|-----------|---------| -| `hybrid` | sqlite-vec available + FTS5 match | Most queries — best precision & recall | -| `vector` | sqlite-vec available, FTS5 returns nothing | Conceptual/semantic queries | -| `fts5` | Part of hybrid pipeline | Keyword-heavy queries | -| `keyword` | sqlite-vec unavailable | Fallback — exact word matching only | +| Method | When Used | Best For | +| --------- | ------------------------------------------ | -------------------------------------- | +| `hybrid` | sqlite-vec available + FTS5 match | Most queries — best precision & recall | +| `vector` | sqlite-vec available, FTS5 returns nothing | Conceptual/semantic queries | +| `fts5` | Part of hybrid pipeline | Keyword-heavy queries | +| `keyword` | sqlite-vec unavailable | Fallback — exact word matching only | The active method is returned in each result's `scoreExplanation.method` field. @@ -68,12 +76,17 @@ Every search result includes a `scoreExplanation` object: ## Tuning Search -| Option | Default | Effect | -|--------|---------|--------| -| `limit` | 10 | Results per page | -| `offset` | 0 | Pagination offset | -| `maxChunksPerDocument` | unlimited | Max chunks returned per document | -| `contextChunks` | 0 | Adjacent chunks to include for context (max 2) | -| `diversity` | 0 | MMR diversity factor (0 = relevance only, 1 = max diversity) | -| `minRating` | none | Filter by minimum avg document rating | -| `tags` | none | Filter by document tags (AND logic) | +These are the parameters of the `search` operation: the MCP `search` tool, `GET /api/v1/search` and `scope.search()` use these names. The CLI flag is in the last column. + +| Option | Default | Effect | CLI flag | +| ---------------------- | --------- | ------------------------------------------------------------------------- | --------------- | +| `limit` | 10 | Results per page (max 100) | `-n, --limit` | +| `offset` | 0 | Pagination offset | `--offset` | +| `maxChunksPerDocument` | unlimited | Max chunks returned per document | `--max-per-doc` | +| `contextChunks` | 0 | Adjacent chunks to include for context (max 2) | `--context` | +| `diversity` | 0 | MMR diversity factor (0 = relevance only, 1 = max diversity) | — | +| `minRating` | none | Filter by minimum avg document rating | `--min-rating` | +| `tags` | none | Filter by document tags (AND logic) | `--tags` | +| `relatedTo` | none | Find content similar to a document or chunk ID instead of running a query | `--related` | + +See the [MCP Tools Reference](/reference/mcp-tools#search) and the [CLI reference](/reference/cli#libscope-search) for all parameters. diff --git a/docs/guide/knowledge-packs.md b/docs/guide/knowledge-packs.md index e9a7dbd5..5306dad3 100644 --- a/docs/guide/knowledge-packs.md +++ b/docs/guide/knowledge-packs.md @@ -1,24 +1,27 @@ # Knowledge Packs -Knowledge packs are portable collections of documents you can share, install, and version. Think of them like npm packages, but for documentation. +Knowledge packs are portable collections of documents you can share, install, and version. Think of them like npm packages, but for documentation. A pack is a `.json` file, or a gzip-compressed `.json.gz` file. ## Installing Packs ```bash # From a local file libscope pack install ./react-docs.json +libscope pack install ./react-docs.json.gz -# From a registry -libscope pack install react-docs --registry https://registry.example.com +# From a pack registry (see Pack Registries) +libscope pack install react-docs +libscope pack install react-docs@1.2.0 +libscope pack install react-docs --registry official # List installed packs libscope pack list -# List available packs from a registry +# List the packs available in your registries libscope pack list --available ``` -When you install a pack, its documents get indexed into your knowledge base just like any other content. They show up in search results and can be queried via RAG. +Pack names are looked up in the git [pack registries](/guide/pack-registries) you added with `libscope registry add`. When you install a pack, its documents get indexed into your knowledge base just like any other content. They show up in search results and can be queried via RAG. ## Creating Packs @@ -28,12 +31,12 @@ You can export documents from your knowledge base as a pack: libscope pack create \ --name "react-docs" \ --topic react \ - --version 1.0.0 \ + --pack-version 1.0.0 \ --description "React documentation" \ --author "team" ``` -This creates a JSON file containing the documents, their metadata, and topic assignments. Share it with your team, commit it to a repo, or publish it to a registry. +This creates a JSON file containing the documents, their metadata, and topic assignments. Share it with your team, commit it to a repo, or [publish it to a registry](/guide/pack-registries#publishing-packs). ## Removing Packs @@ -45,9 +48,9 @@ This removes the pack's documents from your knowledge base. ## MCP Usage -Packs are also available via MCP tools: +Packs are also available in the MCP admin toolset (see [MCP tools](/reference/mcp-tools)): -- `install-pack` — install from registry or file path -- `list-packs` — list installed or available packs +- `install-pack` — install a pack by name (or `name@version`) from the configured registries. Local pack files can be installed only with the CLI or the Node.js API. +- `list-packs` — list installed packs, or with `available: true` the packs in the configured registries Your AI assistant can install packs directly when it needs documentation for a specific library. diff --git a/docs/guide/lite.md b/docs/guide/lite.md index 1fc75476..cfd61a8b 100644 --- a/docs/guide/lite.md +++ b/docs/guide/lite.md @@ -1,33 +1,18 @@ # LibScope Lite — Embedded Semantic Search -`libscope/lite` is a lightweight, embeddable version of LibScope designed to be imported directly into any Node.js application. Instead of running a standalone CLI process or an MCP server, you call `index()` and `search()` programmatically from your own code. +`libscope/lite` puts LibScope inside your own Node.js application. You call `add()`, `search()` and `ask()` from your code. There is no CLI process and no server. -## When to Use LibScope Lite +`createLite()` returns a normal [`LibScope`](/guide/programmatic-usage) object. The methods and results are the same as `LibScope.create()`. The differences: -Use `libscope/lite` when you need to: +- It does not read config files, `~/.libscope/secrets.json` or `LIBSCOPE_*` config variables. Settings come only from the options that you pass. +- You must name the database (`dbPath` or `db`). +- `libscope/lite` does not load the CLI, the MCP server or the connectors when you import it. -- **Embed semantic search into another application** — e.g., a custom MCP server, a VS Code extension, a CI/CD tool -- **Avoid spawning subprocesses** — no CLI execution, no HTTP server required -- **Control the database lifecycle** — pass `:memory:` for ephemeral sessions or a file path for persistent cross-session reuse -- **Search across code** — tree-sitter powered chunking splits source files at function and class boundaries +## When to use Lite -The primary use case driving this feature: a Bitbucket MCP server that wants semantic search over repository files and Jira/Confluence pages. On repository connect, it calls `indexBatch(repoFiles)` to build a local index. On PR review, it calls `getContext(question)` to retrieve the top-K relevant chunks and inject them into its LLM prompt — replacing 50 raw files of context with 5 highly-relevant chunks. - -## What Lite Does NOT Include - -`libscope/lite` intentionally omits the full-LibScope surface area: - -| Feature | Full `libscope` | `libscope/lite` | -|---|---|---| -| Semantic search | ✅ | ✅ | -| RAG (ask/stream) | ✅ | ✅ | -| Code-aware chunking | ❌ | ✅ | -| CLI commands | ✅ | ❌ | -| MCP server | ✅ | ❌ | -| Connectors (Notion, Slack…) | ✅ | ❌ | -| Topics & packs | ✅ | ❌ | -| Webhooks & registry | ✅ | ❌ | -| Web dashboard | ✅ | ❌ | +- To add semantic search to another application, for example a custom MCP server, an editor extension or a CI tool. +- To control the database: `":memory:"` for a temporary index, or a file for an index that you keep. +- To index source code with tree-sitter chunks at function and class boundaries. ## Installation @@ -35,392 +20,193 @@ The primary use case driving this feature: a Bitbucket MCP server that wants sem npm install libscope ``` -For code indexing, also install the optional peer dependencies for the languages you need: +For code chunking, also install tree-sitter and the grammars that you need (optional peer dependencies). Use these versions: npm refuses the latest grammar releases next to libscope. ```bash -# Core parser (required for any language) -npm install tree-sitter - -# Install grammars for the languages you need -npm install tree-sitter-typescript tree-sitter-javascript tree-sitter-python -npm install tree-sitter-c-sharp tree-sitter-cpp tree-sitter-c tree-sitter-go +npm install tree-sitter@0.21 +npm install tree-sitter-typescript@0.21 tree-sitter-javascript@0.21 tree-sitter-python@0.21 +npm install tree-sitter-c-sharp@0.21 tree-sitter-cpp@0.22 tree-sitter-c@0.21 tree-sitter-go@0.21 ``` -These are optional — if not installed, code chunking is unavailable but all other features work. +If they are not installed, code is chunked with the built-in text chunker. -## Quick Start +## Quick start ```ts -import { LibScopeLite } from "libscope/lite"; +import { createLite } from "libscope/lite"; -const lite = new LibScopeLite({ dbPath: ":memory:" }); +const scope = createLite({ + dbPath: ":memory:", + config: { llm: { provider: "passthrough" } }, // ask() returns context for your own LLM +}); -// Index some documents -await lite.indexBatch([ - { title: "Auth Guide", content: "Use OAuth2 for all API access. Tokens expire after 1 hour." }, - { title: "Deploy Guide", content: "Deploy to Kubernetes using Helm charts. Set replicas: 3." }, -], { concurrency: 4 }); +await scope.add({ + title: "Auth Guide", + content: "Use OAuth2 for all API access. Tokens expire after 1 hour.", +}); +await scope.add({ + title: "Deploy Guide", + content: "Deploy to Kubernetes using Helm charts. Set replicas: 3.", +}); -// Hybrid vector + FTS5 search -const results = await lite.search("how to authenticate"); -console.log(results[0]?.title); // "Auth Guide" +const { items } = await scope.search("how to authenticate"); +console.log(items[0]?.title, items[0]?.documentId); // "Auth Guide", "" -// RAG context retrieval (for external LLMs) -const context = await lite.getContext("How do I authenticate API requests?"); -// Returns a formatted context string ready to inject into an LLM prompt +const context = await scope.ask("How do I authenticate API requests?"); +if (context.mode === "context") { + // Put context.contextPrompt into your LLM prompt. context.sources lists the chunks. +} -lite.close(); +scope.close(); ``` -## Constructor Options +## Options ```ts -new LibScopeLite(opts?: LiteOptions) +createLite(options: LiteOptions): LibScope ``` -| Option | Type | Default | Description | -|---|---|---|---| -| `dbPath` | `string` | `~/.libscope/lite.db` | SQLite database path. Use `":memory:"` for in-memory. | -| `db` | `Database` | — | Inject an existing `better-sqlite3` instance. When provided, `dbPath` is ignored and no migrations or schema setup are run. | -| `provider` | `EmbeddingProvider` | Local (all-MiniLM-L6-v2) | Embedding provider to use for indexing and search. | -| `llmProvider` | `LlmProvider` | — | LLM provider for `ask()` and `askStream()`. Required to use those methods. | - -### Persistent vs In-Memory Database - -```ts -// In-memory — data is lost when the process exits (good for one-off tasks) -const lite = new LibScopeLite({ dbPath: ":memory:" }); - -// File-backed — persists across sessions (good for long-lived indexes) -const lite = new LibScopeLite({ dbPath: "/data/my-project.db" }); -``` - -### Embedding Providers - -By default, LibScope Lite uses the local `all-MiniLM-L6-v2` model (downloads ~80 MB on first use). To use OpenAI or Ollama: - -```ts -import { LibScopeLite } from "libscope/lite"; -import { createEmbeddingProvider } from "libscope"; - -const provider = createEmbeddingProvider({ - embedding: { provider: "openai", model: "text-embedding-3-small" }, -}); - -const lite = new LibScopeLite({ provider }); -``` +| Option | Type | Default | Description | +| ------------- | ------------------- | ------------------------ | ------------------------------------------------------------------------------------- | +| `dbPath` | `string` | — | SQLite file, or `":memory:"`. Give `dbPath` or `db`. | +| `db` | `Database` | — | An open `better-sqlite3` database. It is migrated. `close()` leaves it open. | +| `provider` | `EmbeddingProvider` | Local (all-MiniLM-L6-v2) | Embedding provider instance. | +| `llmProvider` | `LlmProvider` | — | LLM for `ask()` and `askStream()`. | +| `chunker` | `Chunker` | — | Custom chunker, for example `createCodeChunker()`. | +| `config` | `ConfigOverrides` | — | Config values over the defaults, for example `{ embedding: { provider: "openai" } }`. | -## Indexing +`createLite(options)` is the same as `LibScope.create({ ...options, useConfigFile: false })`. -### `index(docs)` +### Embedding providers -Index an array of pre-parsed documents: +The default is the local `all-MiniLM-L6-v2` model (about 80 MB, downloaded on first use). To use OpenAI or Ollama, set it in `config`: ```ts -await lite.index([ - { - title: "Getting Started", - content: "# Introduction\n\nThis guide covers the basics...", - library: "my-api", - version: "2.0", - url: "https://docs.example.com/getting-started", +const scope = createLite({ + dbPath: "/data/index.db", + config: { + embedding: { provider: "openai", model: "text-embedding-3-small" }, + openai: { apiKey: process.env.OPENAI_API_KEY }, }, -]); -``` - -**`LiteDoc` fields:** - -| Field | Type | Description | -|---|---|---| -| `title` | `string` | Document title (required) | -| `content` | `string` | Document text content (required) | -| `url` | `string?` | Source URL for deduplication and attribution | -| `library` | `string?` | Library namespace for scoped search | -| `version` | `string?` | Library version | -| `sourceType` | `string?` | `"manual"` (default), `"library"`, `"topic"`, or `"model-generated"` | -| `topicId` | `string?` | Topic ID to associate the document with | -| `language` | `string?` | Language alias for code-aware tree-sitter chunking (e.g. `"typescript"`, `"cpp"`, `"go"`). When set and tree-sitter is available, chunks at function/class boundaries instead of text boundaries. | - -### `indexRaw(input)` - -Index from a raw source — a file path, URL, buffer, or plain text. LibScope Lite normalizes the input using the same parser pipeline as full LibScope: - -```ts -// Index a local file (auto-detects format from extension) -const docId = await lite.indexRaw({ type: "file", path: "./docs/guide.pdf" }); - -// Fetch and index a URL -const docId = await lite.indexRaw({ type: "url", url: "https://docs.example.com/guide" }); - -// Index raw text -const docId = await lite.indexRaw({ type: "text", title: "Notes", content: "..." }); - -// Index from a buffer (e.g., uploaded file) -const docId = await lite.indexRaw({ - type: "buffer", - buffer: fileBuffer, - filename: "report.docx", - title: "Q4 Report", }); ``` -**Supported formats:** Markdown, plain text, HTML, PDF (requires `pdf-parse`), DOCX (requires `mammoth`), EPUB (requires `epub2`), PPTX (requires `pizzip`), CSV, JSON, YAML. - -### `indexBatch(docs, opts)` +Or pass your own object that implements `EmbeddingProvider` as `provider`. -Index multiple documents with concurrency control: +## Add content ```ts -await lite.indexBatch(repoFiles, { concurrency: 4 }); +await scope.add({ title: "Notes", content: "...", library: "my-repo", tags: ["notes"] }); +await scope.add("./docs/guide.md"); // a local file (PDF, DOCX, HTML, Markdown, ...) +await scope.add({ source: "https://example.com/page" }); ``` -`concurrency` controls how many documents are embedded in parallel. A value of 4–8 is recommended for most systems. Each document's embeddings are computed concurrently but each database write is atomic. +To add many documents, run the calls in parallel or in a loop. Each result lists the new `documentId` values. -**Pattern for large repos:** +`normalizeRawInput()` turns a file, an in-memory buffer, a URL or text into `{ title, content }`: ```ts -const files = await glob("src/**/*.ts"); -const docs = await Promise.all( - files.map(async (path) => ({ - title: path, - content: await fs.readFile(path, "utf8"), - sourceType: "library" as const, - })) -); - -await lite.indexBatch(docs, { concurrency: 8 }); -``` - -## Searching - -### `search(query, opts?)` - -Hybrid vector + FTS5 search — the same engine used by full LibScope: - -```ts -const results = await lite.search("OAuth2 token refresh", { - limit: 5, // max results (default: 10) - library: "api", // scope to a library - tags: ["auth"], // filter by tags - diversity: 0.3, // MMR reranking (0 = pure relevance, 1 = max diversity) -}); +import { normalizeRawInput } from "libscope/lite"; -for (const result of results) { - console.log(result.title, result.score); - console.log(result.content); // the matching chunk text -} +const doc = await normalizeRawInput({ type: "buffer", buffer: upload, filename: "report.pdf" }); +await scope.add({ title: doc.title, content: doc.content }); ``` -**`LiteSearchResult` fields:** - -| Field | Type | Description | -|---|---|---| -| `docId` | `string` | Document ID | -| `chunkId` | `string` | Chunk ID within the document | -| `title` | `string` | Document title | -| `content` | `string` | Chunk text | -| `score` | `number` | Relevance score (higher is better) | -| `url` | `string \| null` | Source URL if set at index time | - -## RAG - -### `getContext(question, opts?)` - -Retrieve context without running an LLM — useful when you want to inject the context into your own LLM prompt: +## Search and ask ```ts -const context = await lite.getContext("How does the event loop work?", { - topK: 5, // number of chunks to retrieve (default: 5) - library: "node", // optional scope -}); - -// context is a formatted string you can inject into any LLM prompt -const prompt = `Answer based only on this context:\n\n${context}\n\nQuestion: ...`; +const { items, total } = await scope.search({ query: "deploy", library: "my-repo", limit: 5 }); ``` -This is the primary method for agent-to-agent integration patterns — your orchestrating LLM calls `getContext()` and injects the result into its prompt rather than managing a separate RAG system. - -### `ask(question, opts?)` - -Full RAG with an LLM completing the response (requires `llmProvider` in constructor or opts): +`ask()` returns `{ mode: "answer", answer, sources, model }` when there is an LLM (`llmProvider`, or `config.llm`). With `config: { llm: { provider: "passthrough" } }` it returns `{ mode: "context", contextPrompt, sources }`. With neither, it throws a `ConfigError`. ```ts -import { LibScopeLite } from "libscope/lite"; -import { createLlmProvider } from "libscope"; - -const lite = new LibScopeLite({ - llmProvider: createLlmProvider({ llm: { provider: "openai", model: "gpt-4o-mini" } }), +const scope = createLite({ dbPath: ":memory:", llmProvider: myLlm }); +const result = await scope.ask({ + question: "How do I deploy?", + topK: 5, + systemPrompt: "Be brief.", }); -const answer = await lite.ask("How do I configure rate limiting?", { topK: 5 }); -console.log(answer); // string answer from LLM -``` - -### `askStream(question, opts?)` - -Streaming version of `ask()` — returns an `AsyncGenerator` of token chunks: - -```ts -for await (const token of lite.askStream("Explain the deployment process")) { - process.stdout.write(token); +for await (const event of scope.askStream("How do I deploy?")) { + if ("token" in event) process.stdout.write(event.token); } ``` -The LLM provider must support streaming. Providers that don't expose a `completeStream()` method will throw a clear error. +## Code indexing -## Code Indexing +`createCodeChunker()` splits code at function and class boundaries with tree-sitter. It finds the language from the file extension of the source path, URL or title, or uses the `language` that you give. -LibScope Lite can split source code files at function and class boundaries using tree-sitter rather than plain text chunking. The preferred way to enable this is to set the `language` field on a `LiteDoc` — no extra imports or chunking steps required. +```ts +import { createLite, createCodeChunker } from "libscope/lite"; -### Preferred: set `language` on `LiteDoc` +const scope = createLite({ dbPath: "/data/repo.db", chunker: createCodeChunker() }); -```ts -// Preferred: just set language on LiteDoc — chunking is automatic -await lite.index([ - { - title: "src/auth.cpp", - content: fileContent, - library: "my-repo", - language: "cpp", // enables tree-sitter chunking at function boundaries - }, -]); +await scope.add({ title: "src/auth.ts", content: source, library: "my-repo" }); +// or one language for everything: +createCodeChunker({ language: "python" }); ``` -Setting `language` on a `LiteDoc` automatically triggers code-aware tree-sitter chunking. This is the preferred approach over using `TreeSitterChunker` directly. If tree-sitter is not installed or parsing fails, indexing falls back silently to the standard text chunker. +Documents in other languages, and code when tree-sitter is not installed, use the built-in chunker. -Supported languages and their extension aliases: +Supported languages and aliases: -| Language | Aliases | -|---|---| -| `typescript` | `ts`, `tsx` | +| Language | Aliases | +| ------------ | ------------------------- | +| `typescript` | `ts`, `tsx` | | `javascript` | `js`, `jsx`, `mjs`, `cjs` | -| `python` | `py` | -| `csharp` | `cs` | -| `cpp` | `cc`, `cxx`, `hpp`, `h` | -| `c` | — | -| `go` | — | +| `python` | `py` | +| `csharp` | `cs` | +| `cpp` | `cc`, `cxx`, `hpp`, `h` | +| `c` | — | +| `go` | — | -### Advanced: using `TreeSitterChunker` directly +To get the raw chunks with line numbers, use `TreeSitterChunker` directly. A `chunker` can also be your own function: it receives `{ content, title, source }` and returns an array of chunk strings, or `undefined` to use the built-in chunker. -Direct use of `TreeSitterChunker` is rarely needed when using `LibScopeLite` — setting `language` on `LiteDoc` covers most cases. Use `TreeSitterChunker` directly only when you need access to the raw `CodeChunk` objects (e.g., to extract line numbers for display, filter by node type, or build custom chunk titles): +## Delete, rate, close ```ts -import { LibScopeLite } from "libscope/lite"; -import { TreeSitterChunker } from "libscope/lite"; - -const chunker = new TreeSitterChunker(); -const lite = new LibScopeLite({ dbPath: ":memory:" }); - -// Check if a language is supported before chunking -if (chunker.supports("typescript")) { - const source = await fs.readFile("src/auth.ts", "utf8"); - const chunks = await chunker.chunk(source, "typescript"); - - // Each chunk is a function or class with 1-based line numbers - for (const chunk of chunks) { - await lite.index([{ - title: `auth.ts:${chunk.startLine}-${chunk.endLine} (${chunk.nodeType})`, - content: chunk.content, - library: "src", - }]); - } -} -``` - -See [Code Indexing](/guide/code-indexing) for the full guide including supported languages, chunk shape, and large-file strategies. +// Delete all documents of a library (at most 1000 per call; repeat until affected is 0). +let deleted; +do { + deleted = await scope.admin.bulkDelete({ library: "my-repo" }); +} while (deleted.affected > 0); -### `deleteByLibrary(library)` +await scope.docs.rate({ documentId, rating: 5 }); // ratings boost later search results -Delete all indexed documents belonging to a library namespace. Useful before a full reindex to avoid stale chunks accumulating: - -```ts -// Clear all previously indexed content for this repo, then reindex -lite.deleteByLibrary("my-repo"); -await lite.indexBatch(freshDocs, { concurrency: 4 }); +scope.close(); ``` -The library name matches the `library` field set at index time. Deletion is batched internally and loops until all matching documents are removed. +Create one instance for a long-running service. In a script, close it in a `finally` block. -## Feedback - -### `rate(docId, score)` - -Record a quality signal for a document (score 1–5): - -```ts -const results = await lite.search("deployment process"); -const docId = results[0]?.docId; -if (docId) { - lite.rate(docId, 5); // this result was highly relevant -} -``` - -Ratings feed into subsequent searches — highly-rated documents get boosted in results over time. - -## Lifecycle - -### `close()` - -Always close the database when done: +## Example: an MCP server with a repository index ```ts -lite.close(); -``` - -For long-running services, create one `LibScopeLite` instance and reuse it for the lifetime of the service. For one-off scripts, close in a `finally` block: - -```ts -const lite = new LibScopeLite({ dbPath: "/data/repo.db" }); -try { - await lite.indexBatch(docs, { concurrency: 4 }); - const results = await lite.search(query); - // ... -} finally { - lite.close(); -} -``` - -## Integration Pattern: External MCP Server +import { createLite, createCodeChunker } from "libscope/lite"; -The canonical use case — an MCP server that builds a semantic index over repository files: +const scope = createLite({ + dbPath: path.join(os.homedir(), ".bitbucket-mcp", "index.db"), + chunker: createCodeChunker(), + config: { llm: { provider: "passthrough" } }, +}); -```ts -import { LibScopeLite } from "libscope/lite"; -import { TreeSitterChunker } from "libscope/lite"; - -const DB_PATH = path.join(os.homedir(), ".bitbucket-mcp", "index.db"); -const chunker = new TreeSitterChunker(); -let lite: LibScopeLite; - -// Called when a repo is connected -async function onRepoConnect(repoFiles: { path: string; content: string }[]) { - lite = new LibScopeLite({ dbPath: DB_PATH }); - - const docs = await Promise.all(repoFiles.map(async ({ path, content }) => { - if (chunker.supports(path.split(".").pop() ?? "")) { - // Code-aware chunking for supported languages - const chunks = await chunker.chunk(content, path.split(".").pop()!); - return chunks.map((c) => ({ - title: `${path}:${c.startLine}-${c.endLine}`, - content: c.content, - url: path, - })); - } - return [{ title: path, content, url: path }]; - })); - - await lite.indexBatch(docs.flat(), { concurrency: 4 }); +// When a repository is connected: +async function onRepoConnect(repo: string, files: { path: string; content: string }[]) { + for (const { path, content } of files) { + await scope.add({ title: path, content, library: repo }); + } } -// Called during PR review +// During a pull request review: async function onPrReview(question: string): Promise { - return lite.getContext(question, { topK: 5 }); + const result = await scope.ask({ question, topK: 5 }); + return result.mode === "context" ? result.contextPrompt : result.answer; } ``` -## See Also +## See also -- [Code Indexing Guide](/guide/code-indexing) — tree-sitter chunking in depth -- [LibScope Lite API Reference](/reference/lite-api) — full TypeScript API -- [Programmatic Usage](/guide/programmatic-usage) — full `LibScope` SDK (with connectors, packs, topics) -- [How Search Works](/guide/how-search-works) — hybrid vector + FTS5 explained +- [LibScope Lite API Reference](/reference/lite-api) +- [Programmatic Usage](/guide/programmatic-usage): every `LibScope` method +- [Code Indexing](/guide/code-indexing) +- [How Search Works](/guide/how-search-works) diff --git a/docs/guide/mcp-setup.md b/docs/guide/mcp-setup.md index 6aec26d3..2c05d185 100644 --- a/docs/guide/mcp-setup.md +++ b/docs/guide/mcp-setup.md @@ -78,44 +78,70 @@ If you're using [workspaces](/guide/configuration#workspaces), pass the workspac } ``` +## Admin Tools + +The server starts with 11 core tools. To also give the assistant `sync`, `install-pack`, `list-packs` and `reindex-documents`, enable the `admin` toolset with the `mcp.toolsets` config key: + +```bash +libscope config set mcp.toolsets admin +``` + +You can also set the environment variable `LIBSCOPE_MCP_TOOLSETS` (a comma-separated list) in the server's environment: + +```json +{ + "mcpServers": { + "libscope": { + "command": "npx", + "args": ["-y", "libscope", "serve"], + "env": { "LIBSCOPE_MCP_TOOLSETS": "admin" } + } + } +} +``` + ## Available Tools -Once connected, your AI assistant gets access to all 31 of LibScope's MCP tools. See the [MCP Tools Reference](/reference/mcp-tools) for full parameter details. - -**Search & Q&A** -- **`search-docs`** — semantic search with topic/library/version/rating filters -- **`ask-question`** — RAG Q&A with synthesized answers and source citations - -**Document Management** -- **`submit-document`** — index new content by text or URL -- **`update-document`** — update title, content, or metadata -- **`get-document`** — retrieve a document by ID -- **`list-documents`** — list docs with filters -- **`delete-document`** — remove a document -- **`rate-document`** — rate 1–5 with feedback -- **`suggest-tags`** — auto-suggest tags based on content - -**Organization** -- **`list-topics`** — browse the topic hierarchy -- **`link-documents`** — create cross-references between docs -- **`get-document-links`** — list a document's incoming and outgoing links -- **`delete-link`** — remove a cross-reference - -**Saved Searches** -- **`save-search`** — save a named query with filters -- **`list-saved-searches`** — list saved searches -- **`run-saved-search`** — execute a saved search -- **`delete-saved-search`** — delete a saved search - -**Similar Content** -- **`get-related`** — find chunks semantically similar to a given chunk - -**Connectors** — trigger syncs directly from your AI assistant: -- **`sync-obsidian-vault`**, **`sync-notion`**, **`sync-confluence`**, **`sync-slack`**, **`sync-onenote`** - -**Packs & Maintenance** -- **`install-pack`**, **`list-packs`** — manage knowledge packs -- **`reindex-documents`** — re-embed after switching providers -- **`health-check`** — DB status and doc/chunk counts - -Your AI assistant will call these tools automatically when it needs information from your docs. +See the [MCP Tools Reference](/reference/mcp-tools) for all parameters. + +**Core tools** + +- **`search`**: search by meaning and keywords, or find content related to a document or chunk (`relatedTo`). Filters: topic, library, version, source type, tags, minimum rating. +- **`ask`**: answer a question from the knowledge base. By default (`llm.provider: auto`) the server returns the retrieved context and your assistant writes the answer. Registered only when an answer is possible. +- **`get-document`**: read a document with its tags, links and ratings. Long content is paged. +- **`list-documents`**: list documents with filters, `offset` and a total count. +- **`overview`**: document, chunk and topic counts, topics, installed packs, the embedding model of the index, and health. +- **`submit-document`**: add inline content, a web page, a site crawl (`spider: true`) or a public repository URL. +- **`update-document`**: change the title, content, metadata or tags. +- **`delete-document`**: delete a document. +- **`rate-document`**: rate a document 1-5, with optional feedback or a correction. +- **`link-documents`**: create (`action: "create"`) or delete (`action: "delete"`) a link between documents. +- **`task`**: status, cancel or list of background tasks started with `async: true`. + +**Admin tools** (with `mcp.toolsets` set to `admin` or `all`) + +- **`sync`**: sync one saved connector connection, or all of them. Set up connections with `libscope connect`; the tool does not accept credentials. +- **`install-pack`**, **`list-packs`**: manage knowledge packs. +- **`reindex-documents`**: re-embed chunks after you change the embedding model. + +Your AI assistant calls these tools when it needs information from your docs. The server also sends the assistant short instructions that describe the search, read and rate workflow. + +## Logs + +The MCP server writes JSON-RPC messages to stdout and its logs to stderr. Set the log level with `logging.level` in the config file, `LIBSCOPE_LOGGING_LEVEL`, or `libscope serve --log-level debug` (`--verbose` also sets `debug`). + +## Embedding the Server + +`libscope/mcp` exports `createMcpServer()`. Importing the module does not start a server. + +```ts +import { createMcpServer, runStdioServer } from "libscope/mcp"; + +// Build a server and connect your own transport. +const { server, tools, close } = createMcpServer({ workspace: "my-project", toolsets: ["admin"] }); + +// Or serve over stdio, the same as `libscope serve`. +await runStdioServer({ workspace: "my-project" }); +``` + +`createMcpServer` accepts the same options as LibScope's startup (`workspace`, `dbPath`, `config`, `provider`) or an existing operation context (`ctx`). `close()` closes the server and the database that it opened. diff --git a/docs/guide/pack-registries.md b/docs/guide/pack-registries.md index 0323c032..74a7cbed 100644 --- a/docs/guide/pack-registries.md +++ b/docs/guide/pack-registries.md @@ -7,16 +7,16 @@ Authentication is handled entirely by git. If you have SSH keys or an HTTPS cred ## Adding a Registry ```bash -# Add a public registry +# Add a public registry (the name "libscope-registry" comes from the URL) libscope registry add https://github.com/org/libscope-registry.git -# Add with a custom alias and priority -libscope registry add git@github.com:team/internal-packs.git --name team-packs --priority 5 +# Add with a custom name +libscope registry add git@github.com:team/internal-packs.git --name team-packs -# Add with auto-sync every 24 hours -libscope registry add https://github.com/org/registry.git --sync-interval 86400 +# Add a registry on a local or shared disk +libscope registry add file:///srv/libscope-packs.git -# Add without cloning immediately +# Add without cloning now libscope registry add https://github.com/org/registry.git --no-sync # List configured registries @@ -26,7 +26,7 @@ libscope registry list libscope registry remove team-packs ``` -On first add, libscope clones the registry's index locally to `~/.libscope/registries//`. Subsequent syncs fetch only changes. +When you add a registry, libscope clones it to `~/.libscope/registries//`. Later syncs fetch only changes. ## Searching Registries @@ -35,33 +35,36 @@ On first add, libscope clones the registry's index locally to `~/.libscope/regis libscope registry search "react" # Search a specific registry -libscope registry search "react" -r official +libscope registry search "react" --registry official ``` Results show the pack name, description, tags, latest version, and which registry it came from. ## Installing Packs from a Registry -The existing `pack install` command now resolves packs from your configured registries: +`pack install` finds pack names in your configured registries: ```bash # Install the latest version libscope pack install react-docs # Install a specific version -libscope pack install react-docs --version 1.2.0 -# or libscope pack install react-docs@1.2.0 -# Install from a specific registry (skips conflict resolution) +# Install from a specific registry libscope pack install react-docs --registry official + +# List the packs in all registries +libscope pack list --available ``` -If multiple registries contain a pack with the same name, libscope resolves the conflict by priority (lower `priority` value wins). You can override this with `--registry `. +If more than one registry has a pack with the same name, the install fails and the error names the registries. Run it again with `--registry `. + +Before it installs a pack, libscope verifies the pack file against the checksum in the registry. ### Offline Behavior -If a registry is unreachable during install, libscope falls back to the cached index with a warning. If the registry has never been synced, it tells you to run `libscope registry sync` when online. +Searching, listing and installing read only the local copies of the registries. They do not use the network. A registry that was never synced is skipped with a warning that tells you to run `libscope registry sync`. ## Syncing Registries @@ -73,7 +76,7 @@ libscope registry sync libscope registry sync official ``` -Registries also auto-sync when the local cache is older than the configured `syncInterval` (in seconds). This happens automatically before pack installs when the cache is stale. +Registries are synced only when you run `registry sync` (or `registry add`). If a registry cannot be reached, libscope keeps its local copy and reports it as offline. ## Creating Your Own Registry @@ -89,16 +92,16 @@ This creates a git repo with the correct folder structure (`index.json`, `packs/ ```bash # Publish a pack file to a registry you own -libscope registry publish ./my-pack.json -r my-registry --version 1.0.0 +libscope registry publish ./my-pack.json --registry my-registry --pack-version 1.0.0 -# Auto-bump patch version (from latest in registry) -libscope registry publish ./my-pack.json -r my-registry +# Publish the next patch version (from the latest in the registry) +libscope registry publish ./my-pack.json.gz --registry my-registry # Submit a pack to someone else's registry (creates a feature branch) -libscope registry publish ./my-pack.json -r community --submit +libscope registry publish ./my-pack.json --registry community --submit # Unpublish a specific version -libscope registry unpublish my-pack -r my-registry --version 1.0.0 +libscope registry unpublish my-pack@1.0.0 --registry my-registry ``` Publishing assembles the pack into the registry's folder structure, generates a SHA-256 checksum, updates `index.json` and `pack.json`, and commits + pushes. The `--submit` flag pushes to a `feature/add-` branch instead — you then create a pull request manually. @@ -109,13 +112,12 @@ Every published pack version includes a `checksum.sha256` file. On install, libs ## Versioning -Pack versions follow [semver](https://semver.org/) (e.g. `1.0.0`, `1.2.3`). When you publish without `--version`, the patch version is auto-bumped from the latest. Old versions are preserved in the registry. `pack install` defaults to the latest version unless you specify one. - -## MCP Usage +Pack versions follow [semver](https://semver.org/) (e.g. `1.0.0`, `1.2.3`). When you publish without `--pack-version`, the patch version is auto-bumped from the latest. Old versions are preserved in the registry. `pack install` defaults to the latest version unless you specify one with `name@version`. -Your AI assistant can also work with registries through MCP: +## Other Interfaces -- `install-pack` — install from a registry by name -- `list-packs --available` — browse packs available in registries +- **Node.js API:** `scope.registries.list()`, `.add()`, `.remove()`, `.sync()`, `.search()`, `.create()`, `.publish()`, `.unpublish()`, and `scope.packs.install({ pack, registry })`. +- **REST API:** `GET /api/v1/registries`, `GET /api/v1/registries/search`, `GET /api/v1/packs?available=true` and `POST /api/v1/packs` (read-only access to registries; add, sync and publish with the CLI or the Node.js API). +- **MCP (admin toolset):** `install-pack` installs from a registry by name, and `list-packs` with `available: true` browses the packs in the registries. See the [Registry Reference](/reference/registry) for complete schema details, configuration format, and all CLI flags. diff --git a/docs/guide/programmatic-usage.md b/docs/guide/programmatic-usage.md index 01be9029..6865c895 100644 --- a/docs/guide/programmatic-usage.md +++ b/docs/guide/programmatic-usage.md @@ -1,9 +1,9 @@ # Programmatic Usage -LibScope can be used as a Node.js library via the `LibScope` SDK class. +Use the `LibScope` class to call LibScope from Node.js. Each method calls the same operation as the CLI command, MCP tool and REST route. The parameter names, defaults, validation and results are the same on all of them. -::: tip Looking for embedded / lightweight usage? -[LibScope Lite](/guide/lite) (`libscope/lite`) is a zero-dependency-on-connectors embeddable class with `index()`, `search()`, `getContext()`, and tree-sitter code chunking — designed to be imported directly into external applications without the full CLI/MCP/connector surface area. +::: tip Embedding LibScope in another application? +Use [`libscope/lite`](/guide/lite). `createLite()` returns the same `LibScope` object, but it does not read config files and it uses the database that you name. ::: ## Setup @@ -12,9 +12,24 @@ LibScope can be used as a Node.js library via the `LibScope` SDK class. import { LibScope } from "libscope"; const scope = LibScope.create(); +// ... use scope ... +scope.close(); ``` -You can pass options to `create()`: +`create()` loads the config, opens and migrates the database, and creates the embedding provider. Without options, it opens the database of the active workspace (the same one that the CLI and MCP server use). + +### Options + +| Option | Type | Description | +| --------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `workspace` | `string` | Workspace whose database to open. Default: the active workspace. | +| `dbPath` | `string` | SQLite file. Use `":memory:"` for an in-memory database. Wins over `workspace` and `database.path`. | +| `db` | `Database` | An open `better-sqlite3` database. It is migrated. `close()` does not close it. | +| `config` | `ConfigOverrides` | Values per config section, merged over the loaded config. Example: `{ llm: { provider: "anthropic" } }`. | +| `useConfigFile` | `boolean` | `false`: do not read config files, `secrets.json` or `LIBSCOPE_*` config variables. Start from the defaults and `config`. | +| `provider` | `EmbeddingProvider` | Embedding provider instance to use instead of the configured one. | +| `llmProvider` | `LlmProvider` | LLM for `ask()` and `askStream()` instead of the configured one. | +| `chunker` | `Chunker` | Splits inline content and local files into chunks instead of the built-in markdown chunker. | ```ts const scope = LibScope.create({ @@ -26,111 +41,154 @@ const scope = LibScope.create({ }); ``` -This initializes the database, runs migrations, and sets up embedding/LLM providers automatically. +## Add content -## Indexing Documents +`add()` takes inline content, a file, a directory, a URL or a GitHub/GitLab repository URL. A string is a source path or URL. ```ts -const doc = await scope.index({ +const result = await scope.add({ title: "Auth Guide", content: "# Authentication\n\nUse OAuth2 for all API access...", library: "my-lib", version: "2.0.0", + tags: ["auth"], }); +console.log(result.documents[0]?.documentId); -console.log(doc.id); // document ID +await scope.add("./docs"); // every supported file in the directory +await scope.add({ source: "https://example.com/docs", spider: true, maxPages: 50 }); +await scope.add("https://github.com/org/repo"); ``` -### Document TTL / Auto-Expiry +The result is `{ kind, documents, errors, skipped }`. Each item in `documents` has `documentId`, `title`, `chunkCount` and `source`. + +To make a document expire, set `expiresAt` (ISO 8601). `scope.admin.pruneExpired()` deletes the expired documents. -Set `expiresAt` to an ISO 8601 timestamp to mark a document for automatic expiry: +## Search ```ts -await scope.index({ - title: "Sprint 42 Notes", - content: "...", - expiresAt: "2026-04-01T00:00:00Z", +const { items, total } = await scope.search({ + query: "how to authenticate", + library: "my-lib", + limit: 10, + diversity: 0.3, // MMR reranking: 0 = relevance only, 1 = most diverse }); -``` -Expired documents are not removed automatically — call `pruneExpiredDocuments()` to clean them up: +for (const r of items) console.log(r.documentId, r.chunkId, r.title, r.score); -```ts -import { pruneExpiredDocuments } from "libscope"; - -// Using the low-level function (requires a db handle) -const { pruned } = pruneExpiredDocuments(db); -console.log(`Removed ${pruned} expired documents`); +// A string is the query. +await scope.search("deployment"); +// Content related to a document or chunk: +await scope.search({ relatedTo: items[0]!.documentId }); ``` -## Searching +All list results have the shape `{ items, total, limit, offset }`. -```ts -const { results } = await scope.search("how to authenticate", { - library: "my-lib", - limit: 10, - diversity: 0.3, // MMR diversity reranking (0 = pure relevance, 1 = max diversity) -}); +## Ask -for (const result of results) { - console.log(result.title, result.score); +```ts +const result = await scope.ask({ question: "How does OAuth2 work?", library: "my-lib", topK: 5 }); +if (result.mode === "answer") { + console.log(result.answer, result.sources); +} else { + // llm.provider is "passthrough": answer from result.contextPrompt yourself. + console.log(result.contextPrompt); } ``` -## Batch Search +`ask()` uses `llmProvider`, or the LLM from `llm.provider` (default `auto`: OpenAI if an OpenAI key is set, else Anthropic if an Anthropic key is set, else Ollama if `llm.url` is set or the embedding provider is `ollama`). If there is no LLM, `ask()` throws a `ConfigError` that tells you what to set. -Run up to 20 search queries concurrently: +Streaming needs an LLM: with `passthrough` or no LLM, `askStream()` throws a `ConfigError`. ```ts -const { results } = await scope.searchBatch([ - { query: "authentication" }, - { query: "deployment", options: { library: "my-lib", limit: 5 } }, -]); - -// Results are keyed by query string -console.log(results["authentication"].results.length); -console.log(results["deployment"].results.length); +for await (const event of scope.askStream("How does OAuth2 work?")) { + if ("token" in event) process.stdout.write(event.token); + else console.log("\nSources:", event.sources); +} ``` -## RAG (Ask Questions) +## Namespaces + +The other operations are grouped in namespaces. Each method takes one input object and returns a promise. The input type is the operation's schema input, so your editor shows each field and its description. + +| Method | Operation | +| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | +| `scope.overview()` | Counts, topics, packs, index, health | +| `docs.get({ documentId, offset?, maxLength? })` | Document with tags, links, ratings | +| `docs.list({ topic?, library?, version?, sourceType?, tags?, limit?, offset? })` | List documents | +| `docs.update({ documentId, title?, content?, library?, version?, url?, topic?, tags? })` | Update a document | +| `docs.delete({ documentId })` | Delete a document | +| `docs.rate({ documentId, rating, chunkId?, feedback?, suggestedCorrection? })` | Rate a document | +| `docs.history({ documentId })`, `docs.rollback({ documentId, version })` | Saved versions | +| `topics.list({ parent? })`, `topics.create({ name, description?, parent? })`, `topics.delete({ topic, deleteDocuments? })` | Topics | +| `tags.add`, `tags.remove` (`{ documentId, tags }`), `tags.list()`, `tags.suggest({ documentId })` | Tags | +| `links.create({ documentId, targetDocumentId, linkType, label? })`, `links.delete({ linkId })`, `links.list({ documentId?, linkType? })`, `links.prerequisites({ documentId })` | Document links | +| `links.graph({ topic?, tag?, threshold?, maxNodes? })` | Knowledge graph (nodes and edges) | +| `searches.save`, `searches.list`, `searches.run`, `searches.delete` | Saved searches | +| `packs.install({ pack, registry? })`, `packs.remove({ pack })`, `packs.list({ available?, registry? })`, `packs.create({ name, ... })` | Knowledge packs | +| `registries.list()`, `registries.add({ url, name? })`, `registries.remove({ name })`, `registries.sync({ name? })`, `registries.search({ query, registry? })`, `registries.create({ path })`, `registries.publish({ file, registry, version? })`, `registries.unpublish({ pack, registry })` | Git pack registries | +| `connectors.list()`, `connectors.sync({ name } \| { all: true })`, `connectors.disconnect({ name })` | Saved connections | +| `tasks.get`, `tasks.cancel`, `tasks.list` | Background tasks | +| `admin.reindex`, `admin.dedupe`, `admin.backup`, `admin.restore`, `admin.pruneExpired`, `admin.bulkDelete`, `admin.bulkRetag`, `admin.bulkMove` | Maintenance | +| `analytics.popular`, `analytics.stale`, `analytics.topQueries`, `analytics.searches` | Search analytics | +| `webhooks.create`, `webhooks.list`, `webhooks.delete`, `webhooks.test` | Webhooks | ```ts -const answer = await scope.ask("How does OAuth2 work?", { - library: "my-lib", - topK: 5, -}); - -console.log(answer.text); -console.log(answer.sources); // cited chunks +const doc = await scope.docs.get({ documentId: "doc-id" }); +await scope.tags.add({ documentId: "doc-id", tags: ["reviewed"] }); +await scope.docs.update({ documentId: "doc-id", topic: "security" }); +const { stats } = await scope.overview(); + +// Knowledge graph of one topic +const graph = await scope.links.graph({ topic: "security", maxNodes: 100 }); +console.log(graph.nodes.length, graph.edges.length); + +// Git pack registries +await scope.registries.add({ url: "https://github.com/org/packs.git", name: "team" }); +const found = await scope.registries.search({ query: "react" }); +await scope.packs.install({ pack: "react-docs", registry: "team" }); ``` -For streaming responses: +See [Pack Registries](/guide/pack-registries) for the registry layout and publishing. + +Every method except `askStream` takes an optional second argument with an `AbortSignal` and a progress callback. The long-running methods (`add`, `admin.reindex`, `admin.dedupe`, `packs.install`, `connectors.sync`) report progress: ```ts -for await (const event of scope.askStream("How does OAuth2 work?")) { - if (event.type === "text") process.stdout.write(event.text); -} +const controller = new AbortController(); +await scope.admin.reindex( + { rebuild: true }, + { signal: controller.signal, onProgress: (p) => console.log(p.done, p.total) }, +); ``` -## Other Operations +### Types + +The package root exports the types for inputs and results: ```ts -// Get stats -const stats = scope.stats(); +import type { LibScopeInput, LibScopeOutput, SearchOutput, AskOutput } from "libscope"; + +type UpdateInput = LibScopeInput<"docs", "update">; +type DocumentView = LibScopeOutput<"docs", "get">; +``` -// List documents -const docs = scope.list({ library: "my-lib" }); +## Errors -// Get a single document -const doc = scope.get("doc-id"); +Invalid input throws `ValidationError`. An unknown document, chunk, topic, link or other resource throws a `NotFoundError` subclass. A missing LLM or API key throws `ConfigError`. All of them extend `LibScopeError` and have a `code`. -// Delete a document -scope.delete("doc-id"); +```ts +import { NotFoundError, ValidationError } from "libscope"; + +try { + await scope.docs.get({ documentId: "missing" }); +} catch (err) { + if (err instanceof NotFoundError) console.log(err.code); // "DOCUMENT_NOT_FOUND" +} ``` -## Cleanup +## Close -Always close the database connection when done: +Close the database when you are done: ```ts scope.close(); diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index d382bd25..db016577 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -9,6 +9,7 @@ **Cause:** `sqlite-vec` requires a native Node.js addon that must be built for your platform. **Fix:** + ```bash npm rebuild sqlite-vec # or @@ -21,11 +22,12 @@ If you're on an unsupported platform, LibScope will fall back to keyword-only se ### Model download fails or hangs -**Symptom:** First run hangs at "Downloading model..." or fails with a network error. +**Symptom:** First run hangs at "Downloading embedding model ... (first run only)..." or fails with a network error. **Cause:** The local embedding model (~80MB) downloads from Hugging Face on first use. **Fix:** + - Check your internet connection - If behind a proxy, set `HTTPS_PROXY` environment variable - To use OpenAI embeddings instead (no download required): `libscope config set embedding.provider openai` @@ -34,26 +36,39 @@ If you're on an unsupported platform, LibScope will fall back to keyword-only se ### Embedding dimension mismatch after switching providers -**Symptom:** Error like `expected N dimensions, got M` when searching after changing the embedding provider. +**Symptom:** LibScope stops with `The vector index () does not match the configured embedding model ()`. Or search logs `Vector index does not match the embedding model, falling back to keyword search`. + +**Cause:** LibScope records the provider, model, and vector size that built the vector index. Different models produce different vectors, so the existing vectors cannot be used with the new model. -**Cause:** Changing embedding providers produces vectors of different dimensions. Existing embeddings in the database are incompatible. +**Fix:** Rebuild the vector index with the configured model: -**Fix:** Re-index all documents after switching providers: ```bash -libscope db reset # clears all indexed content -libscope pack install ... # re-install packs +libscope admin reindex --rebuild ``` +This command drops and recreates the vector table with the size of the new model. Then it re-embeds every chunk. Your documents, topics, and tags are kept. + +### `Expected embedding dimension N, got M` + +**Symptom:** Indexing or search fails with `Expected embedding dimension N, got M from model ""`. + +**Cause:** The model returns vectors of a different size than LibScope expects. LibScope knows the size of common OpenAI and Ollama models. For other models, it uses `embedding.dimensions` from the config, or the size of the first vector the model returns. + +**Fix:** Set `embedding.dimensions` to M in `~/.libscope/config.json` or `.libscope.json`, then run `libscope admin reindex --rebuild`. + +`libscope doctor` shows the configured embedding model and the model the vector index was built with. + --- ## Search Issues ### Search returns no results -1. Check that documents are indexed: `libscope list` +1. Check that documents are indexed: `libscope docs list` (or `libscope admin stats` for counts) 2. Try a simpler query — FTS5 AND logic requires all terms to match 3. Check your filters — `--library`, `--topic`, `--tags` may be too restrictive -4. Run `libscope search "test" --limit 5` to verify basic search works +4. Run `libscope search "test" -n 5` to verify basic search works +5. Run `libscope doctor` to check the database, the embedding model and the vector index ### Results seem irrelevant @@ -63,17 +78,14 @@ libscope pack install ... # re-install packs --- -## API / MCP Issues - -### `401 Unauthorized` from API - -The REST API requires an `X-API-Key` header. Find your key: `libscope config show` +## MCP Issues ### MCP tools not appearing in Claude / Cursor -1. Verify the MCP server is running: `libscope mcp start` -2. Check the MCP config path in your client settings points to the libscope server -3. Restart your AI client after adding the MCP server +1. Check that the server starts: run `libscope serve` in a terminal. It waits for JSON-RPC on stdin; any startup error is printed on stderr. Press Ctrl+C to stop it. +2. Check that your client config runs `npx -y libscope serve` (or `libscope serve`). See [MCP Setup](/guide/mcp-setup). +3. Restart your AI client after you change its MCP config. +4. `ask` is registered only when an LLM or passthrough is available, and `sync`, `install-pack`, `list-packs` and `reindex-documents` only with the admin toolset (`LIBSCOPE_MCP_TOOLSETS=admin`). See the [MCP Tools Reference](/reference/mcp-tools). --- @@ -82,15 +94,21 @@ The REST API requires an `X-API-Key` header. Find your key: `libscope config sho ### Database locked errors LibScope uses SQLite WAL mode which supports concurrent reads but only one writer. If you see lock errors: + - Ensure only one libscope process is running - Check for stuck processes: `ps aux | grep libscope` ### How to reset the database +There is no reset command. To remove all indexed content and keep your config, stop all libscope processes (CLI, MCP server, API server). Then delete the database files of the workspace: + ```bash -libscope db reset # removes all indexed content (keeps config) +rm ~/.libscope/workspaces/default/libscope.db* # replace "default" with your workspace name +libscope doctor --fix # optional: the next command also creates it ``` +`libscope config path` and `libscope doctor` show which files are in use. To replace only the vectors (for example, after you change the embedding model), use `libscope admin reindex --rebuild`. + --- ## Connector Issues @@ -99,13 +117,14 @@ libscope db reset # removes all indexed content (keeps config) - Verify your integration token has access to the pages you want to sync — in Notion, you must explicitly share pages with your integration - Check that the token starts with `secret_` (internal integrations) or is a valid OAuth token -- Try syncing a specific page ID first: `libscope connect notion --token $NOTION_TOKEN` +- Run `libscope connections` to see the last run and its error ### Confluence sync fails with 401 - Ensure you are using an API token (not your password) — generate one at [id.atlassian.com](https://id.atlassian.com/manage-profile/security/api-tokens) - The `--email` flag must match the Atlassian account that owns the token -- Verify the `--url` uses your full Atlassian domain: `https://your-org.atlassian.net` +- Verify the base URL (the `[source]` argument of `connect confluence`) is your full Atlassian domain: `https://your-org.atlassian.net` +- For Confluence Server or Data Center, add `--server` and use a personal access token ### Slack sync fails with `missing_scope` error @@ -120,16 +139,16 @@ Reinstall the app to your workspace after adding scopes. ### OneNote sync prompts for authentication every time -OneNote uses device code auth which caches a refresh token. If the cache is missing or expired: +OneNote uses device code sign-in and saves a refresh token in the connection. If the token is missing or no longer valid: -1. Ensure `ONENOTE_CLIENT_ID` is set to your Azure AD app registration client ID -2. The token is cached in `~/.libscope/connectors/onenote.json` — delete this file to force re-authentication +1. Run `libscope connect onenote --client-id ` with your Azure AD app registration client ID (add `--name` if the connection has another name) +2. The tokens are saved in `~/.libscope/connectors/onenote.json` (or `.json`). `libscope disconnect onenote --keep-documents` deletes it and forces a new sign-in 3. Ensure your app registration has `Notes.Read` (or `Notes.ReadWrite`) permission granted in Azure ### Obsidian sync misses some notes -- Check your `--exclude` patterns — glob patterns use `/` as separator even on Windows -- Obsidian notes in the `.obsidian/` config folder are automatically excluded +- Check your `--exclude` patterns (comma-separated globs) — glob patterns use `/` as separator even on Windows +- The `.obsidian/`, `.trash/` and `templates/` folders are always excluded - Notes without the `.md` extension are skipped; check your vault for unusual extensions --- @@ -138,31 +157,25 @@ OneNote uses device code auth which caches a refresh token. If the cache is miss ### `401 Unauthorized` from API -The REST API requires an `X-API-Key` header for write operations. Retrieve your key: +Authentication is on only when the server process has `LIBSCOPE_API_KEY` set. Then every request must send that value as a bearer token. `libscope config show` does not show this key. ```bash -libscope config show -``` - -Then include it in requests: - -```bash -curl -H "X-API-Key: " http://localhost:3378/api/v1/documents +curl -H "Authorization: Bearer $LIBSCOPE_API_KEY" http://localhost:3378/api/v1/documents ``` ### `429 Too Many Requests` -The API applies rate limiting per IP. If you're hitting limits during bulk operations, use the batch endpoints: +The API allows 120 requests per minute from one address. If you hit the limit during bulk operations: -- `POST /api/v1/batch-search` — up to 20 queries at once -- `POST /api/v1/bulk/delete`, `/bulk/retag`, `/bulk/move` — process many documents in one request +- Use `POST /api/v1/bulk/delete`, `/bulk/retag` and `/bulk/move` to change up to 1000 documents in one request +- Send one `POST /api/v1/documents` with `url` and `spider: true` to crawl a site, instead of one request per page ### CORS errors when calling from a browser -The REST API allows CORS by default for localhost origins. If you're calling from a different origin, you can either: +By default, the REST API allows browser requests from `http://localhost` and `http://localhost:3000`. Write requests from other origins get `403` (`FORBIDDEN_ORIGIN`). If you call it from a different origin, you can: -- Run your frontend on the same host as the API -- Use a reverse proxy that adds the appropriate `Access-Control-Allow-Origin` header +- Run your frontend on the same origin as the API +- Start the server from code with `startApiServer` and list your origin in `corsOrigins` (see the [REST API reference](/reference/rest-api)) --- @@ -175,10 +188,9 @@ Optional parser dependencies may not be installed. Install them: ```bash npm install -g pdf-parse # for .pdf files npm install -g mammoth # for .docx files -npm install -g epub2 # for .epub files ``` -Or install all optional parsers at once: +Or reinstall LibScope with its optional dependencies: ```bash npm install -g libscope --include=optional @@ -191,7 +203,7 @@ If you are fetching from a server with a self-signed certificate: ```bash libscope config set indexing.allowSelfSignedCerts true # or -export LIBSCOPE_ALLOW_SELF_SIGNED_CERTS=true +export LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS=true ``` For internal/private URLs (RFC 1918 address ranges): @@ -199,18 +211,14 @@ For internal/private URLs (RFC 1918 address ranges): ```bash libscope config set indexing.allowPrivateUrls true # or -export LIBSCOPE_ALLOW_PRIVATE_URLS=true +export LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=true ``` ### Import is very slow for large directories -Use `import-batch` with parallelism instead of `import`: - -```bash -libscope import-batch ./docs/ --concurrency 8 --library my-lib -``` - -The default `import` command is sequential. `import-batch` processes files in parallel. +- Limit the files with `--include` and `--exclude` (comma-separated globs): `libscope add ./docs --include "**/*.md" --exclude "node_modules/**"` +- Run `libscope add ./docs --dry-run` first to see which files would be added +- The local embedding model runs on the CPU. An Ollama or OpenAI embedding provider can be faster for large imports --- @@ -218,17 +226,17 @@ The default `import` command is sequential. `import-batch` processes files in pa ### `ask` returns "No LLM provider configured" -The `ask` command requires an LLM provider. Configure one: +The `ask` command needs an LLM. With `llm.provider` set to `auto` (the default), LibScope uses OpenAI if an OpenAI key is set, else Anthropic if an Anthropic key is set, else Ollama if `llm.url` is set or the embedding provider is `ollama`. `libscope doctor` shows which LLM `ask` will use. To choose one: ```bash -# OpenAI +# OpenAI (LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY) libscope config set llm.provider openai export LIBSCOPE_OPENAI_API_KEY=sk-... # Ollama (must be running locally) libscope config set llm.provider ollama -# Anthropic +# Anthropic (LIBSCOPE_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY) libscope config set llm.provider anthropic export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... ``` @@ -236,6 +244,6 @@ export LIBSCOPE_ANTHROPIC_API_KEY=sk-ant-... ### Answers are low quality or hallucinated - Index more relevant documents — the quality of RAG answers depends on what's in the knowledge base -- Increase `--top-k` to retrieve more context chunks: `libscope ask "..." --top-k 10` +- Use more context chunks: `libscope ask "..." -n 10` (default 5) - Switch to a more capable embedding provider (OpenAI `text-embedding-3-small` outperforms the local model) - Switch to a more capable LLM model: `libscope config set llm.model gpt-4o` diff --git a/docs/guide/webhooks.md b/docs/guide/webhooks.md index 8125b184..4d23446b 100644 --- a/docs/guide/webhooks.md +++ b/docs/guide/webhooks.md @@ -1,9 +1,19 @@ # Webhooks -LibScope can send HTTP POST notifications to external URLs when documents are created, updated, or deleted. This lets you integrate LibScope with CI pipelines, Slack bots, or any other HTTP-capable service. +LibScope can send HTTP POST notifications to external URLs when documents are created, updated, deleted, or rated, and when a search runs. This lets you integrate LibScope with CI pipelines, Slack bots, or any other HTTP-capable service. + +Events fire from every surface — CLI, MCP server, REST API, and the SDK — in the process that made the change. ## Creating a Webhook +### Via CLI + +```bash +libscope webhooks create https://hooks.example.com/libscope \ + --events document.created,document.updated,document.deleted \ + --secret my-hmac-secret +``` + ### Via REST API ```bash @@ -16,21 +26,19 @@ curl -X POST http://localhost:3378/api/v1/webhooks \ }' ``` -### Via CLI - -Webhook management is available through the REST API only. Start the API server first: +In the SDK, use `scope.webhooks.create({ url, events, secret })`. Webhooks are not available as MCP tools. -```bash -libscope serve --api --port 3378 -``` +Storing a secret requires the `LIBSCOPE_SECRET_KEY` environment variable. LibScope encrypts the secret at rest with AES-256-GCM. Webhook URLs must use http or https and must not resolve to a private or internal IP address. ## Supported Events -| Event | Fired when | -| ------------------- | ---------------------------------------------- | -| `document.created` | A new document is indexed | -| `document.updated` | A document's content or metadata is updated | -| `document.deleted` | A document is deleted | +| Event | Fired when | +| ------------------ | ------------------------------------------- | +| `document.created` | A new document is indexed | +| `document.updated` | A document's content or metadata is updated | +| `document.deleted` | A document is deleted | +| `document.rated` | A document or chunk is rated | +| `search.executed` | A search runs | ## Payload Format @@ -41,27 +49,31 @@ Every webhook delivery sends a `POST` request with `Content-Type: application/js "event": "document.created", "timestamp": "2026-03-18T12:00:00.000Z", "data": { - "id": "doc_abc123", + "documentId": "3f1c…", "title": "Auth Guide", "library": "my-lib", - "version": "2.0.0", - "topic": "security", - "sourceType": "manual", - "url": null, - "createdAt": "2026-03-18T12:00:00.000Z", - "updatedAt": "2026-03-18T12:00:00.000Z" + "version": "2.0.0" } } ``` -For `document.deleted`, the `data` object contains only `id` and `title`. +The `data` object depends on the event: + +| Event | `data` fields | +| -------------------------------------- | --------------------------------------------- | +| `document.created`, `document.updated` | `documentId`, `title`, `library`?, `version`? | +| `document.deleted` | `documentId` | +| `document.rated` | `documentId`, `rating`, `feedback`? | +| `search.executed` | `query`, `resultCount`, `topicId`? | + +Fields marked `?` are omitted when they have no value. ## Verifying Signatures -When you create a webhook with a `secret`, LibScope signs every delivery with HMAC-SHA256. The signature is sent in the `X-LibScope-Signature` header: +When you create a webhook with a `secret`, LibScope signs every delivery with HMAC-SHA256 over the raw request body. The hex digest is sent in the `X-LibScope-Signature` header: ``` -X-LibScope-Signature: sha256=abc123... +X-LibScope-Signature: 9f86d081884c7d65... ``` To verify in Node.js: @@ -69,19 +81,17 @@ To verify in Node.js: ```typescript import { createHmac, timingSafeEqual } from "crypto"; -function verifySignature(secret: string, body: string, header: string): boolean { - const expected = "sha256=" + createHmac("sha256", secret).update(body).digest("hex"); +function verifySignature(secret: string, rawBody: string, header: string): boolean { + const expected = Buffer.from(createHmac("sha256", secret).update(rawBody).digest("hex")); const received = Buffer.from(header); - const expectedBuf = Buffer.from(expected); - if (received.length !== expectedBuf.length) return false; - return timingSafeEqual(received, expectedBuf); + if (received.length !== expected.length) return false; + return timingSafeEqual(received, expected); } -// Express example -app.post("/webhook", (req, res) => { - const rawBody = JSON.stringify(req.body); // or use express.raw() +// Express example — verify against the raw body, not re-serialized JSON +app.post("/webhook", express.raw({ type: "application/json" }), (req, res) => { const sig = req.headers["x-libscope-signature"] as string; - if (!verifySignature("my-hmac-secret", rawBody, sig)) { + if (!verifySignature("my-hmac-secret", req.body.toString("utf8"), sig)) { return res.status(401).send("Invalid signature"); } // handle event @@ -96,26 +106,31 @@ Use `timingSafeEqual` to prevent timing attacks — never use `===` for comparin Send a test ping to verify your endpoint is reachable: ```bash +libscope webhooks test +# or curl -X POST http://localhost:3378/api/v1/webhooks//test ``` -This sends a `POST` to your webhook URL with `event: "ping"` and no `data` payload. +The ping is a `document.created` event with `data: { "test": true, "message": "Webhook test ping" }`. It is signed and SSRF-checked exactly like a real delivery, and the command prints the HTTP status your endpoint returned. ## Managing Webhooks ```bash # List all webhooks +libscope webhooks list curl http://localhost:3378/api/v1/webhooks # Delete a webhook +libscope webhooks delete curl -X DELETE http://localhost:3378/api/v1/webhooks/ ``` ## Delivery Behavior -- Deliveries are attempted synchronously as part of the request that triggered the event -- Failed deliveries (non-2xx response or network error) are logged but not retried automatically -- Webhook secrets are stored hashed and cannot be retrieved after creation — store them securely +- Deliveries run in the background. They never delay or fail the operation that triggered the event. +- Before each delivery, LibScope resolves the URL again and refuses private or internal IPs. Redirects are refused. Each request times out after 5 seconds. +- Failed deliveries (non-2xx response, network error, or SSRF rejection) are logged and counted, but not retried. After 10 consecutive failures the webhook is deactivated. A successful delivery resets the count. +- A short-lived CLI command can exit before a slow endpoint responds. The delivery is still sent, but its result may not be recorded. ## Example: Notify Slack on New Documents @@ -146,10 +161,8 @@ app.post("/relay", async (req, res) => { app.listen(4000); ``` -Then register the relay as a LibScope webhook: +Deploy the relay at a public address, then register it as a LibScope webhook (localhost and private addresses are rejected): ```bash -curl -X POST http://localhost:3378/api/v1/webhooks \ - -H "Content-Type: application/json" \ - -d '{"url": "http://localhost:4000/relay", "events": ["document.created"]}' +libscope webhooks create https://relay.example.com/relay --events document.created ``` diff --git a/docs/index.md b/docs/index.md index fea84d38..dce51613 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,13 +16,13 @@ hero: features: - icon: 🔍 title: Semantic Search - details: Vector similarity search powered by sqlite-vec, with FTS5 full-text fallback. Find what you need, even when you don't know the exact words. + details: Hybrid search that combines vector similarity (sqlite-vec) with FTS5 full-text search. Find what you need, even when you don't know the exact words. - icon: 🤖 title: MCP Integration - details: 31 tools for AI assistants out of the box. Works with Claude, Cursor, VS Code, and any MCP-compatible client. + details: 11 core tools for AI assistants (search, ask, read, add, rate, link), plus an optional admin toolset. Works with Claude, Cursor, VS Code, and any MCP-compatible client. - icon: 🔗 title: Connectors - details: Pull in docs from Obsidian, Notion, Confluence, Slack, OneNote, and GitHub/GitLab. Keep everything in one place. + details: Pull in docs from Obsidian, Notion, Confluence, Slack, OneNote, documentation sites, and GitHub/GitLab repositories. Keep everything in one place. - icon: 📦 title: Knowledge Packs details: Install pre-built documentation bundles or create your own. Share curated collections with your team. diff --git a/docs/migration-v2.md b/docs/migration-v2.md new file mode 100644 index 00000000..1eb2edac --- /dev/null +++ b/docs/migration-v2.md @@ -0,0 +1,516 @@ +# Migrating to LibScope 2.0 + +LibScope 2.0 removes old names without aliases. This page lists every removed or renamed name and what to use instead. Your indexed data is not changed by the upgrade. + +- [Configuration](#configuration): config keys, environment variables, API keys +- [Library API](#library-api): package root exports, the `LibScope` class, `libscope/lite` +- [MCP](#mcp): server startup, renamed, merged and removed tools +- [REST API](#rest-api): routes, request fields, responses, CORS +- [CLI](#cli): commands, options, behavior +- [Packs and registries](#packs-and-registries): the URL registry is removed +- [Client SDKs](#client-sdks): Python and Go clients +- [Saved searches](#saved-searches): filter field names +- [Docker image](#docker-image): command, port, data volume + +The references list the 2.0 names: [CLI](reference/cli.md), [MCP tools](reference/mcp-tools.md), [REST API](reference/rest-api.md) and [configuration](reference/configuration.md). + +## Configuration + +### Config keys + +Edit `~/.libscope/config.json` and `.libscope.json`, or run `libscope config set `. LibScope 2.0 ignores the old keys. + +| Old key (1.x) | New key (2.0) | Notes | +| ------------------------ | ------------------ | ------------------------------------------------------------------------------------------------------ | +| `embedding.ollamaUrl` | `embedding.url` | | +| `embedding.ollamaModel` | `embedding.model` | `embedding.model` applies to the selected `embedding.provider`. | +| `embedding.openaiModel` | `embedding.model` | Same as above. | +| `embedding.openaiApiKey` | `openai.apiKey` | One OpenAI key for embeddings and the LLM. Stored in `~/.libscope/secrets.json`, not in `config.json`. | +| `llm.ollamaUrl` | `llm.url` | When unset, the Ollama LLM uses `embedding.url`, then `http://localhost:11434`. | +| `llm.openaiApiKey` | `openai.apiKey` | Same key as for embeddings. | +| `llm.anthropicApiKey` | `anthropic.apiKey` | Stored in `~/.libscope/secrets.json`. | + +### Environment variables + +| Old variable (1.x) | New variable (2.0) | +| ---------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `LIBSCOPE_OLLAMA_URL` | `LIBSCOPE_EMBEDDING_URL` (and `LIBSCOPE_LLM_URL` for the LLM, which falls back to `LIBSCOPE_EMBEDDING_URL`) | +| `LIBSCOPE_OLLAMA_MODEL` | `LIBSCOPE_EMBEDDING_MODEL` | +| `LIBSCOPE_ALLOW_PRIVATE_URLS` | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | +| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | + +These variables do not change: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_LLM_PROVIDER`, `LIBSCOPE_LLM_MODEL`, `LIBSCOPE_OPENAI_API_KEY`, `OPENAI_API_KEY`, `LIBSCOPE_ANTHROPIC_API_KEY`, `ANTHROPIC_API_KEY`. Every config key now has a variable named `LIBSCOPE_
_`. See the [configuration reference](reference/configuration.md). + +### Behavior changes + +| 1.x | 2.0 | +| ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| API keys could be stored in `config.json` or `.libscope.json`. | API keys are read only from environment variables and `~/.libscope/secrets.json` (mode `0600`). Keys in `config.json` or `.libscope.json` are ignored with a warning. Move them with `libscope config set openai.apiKey ` or `libscope config set anthropic.apiKey `. | +| `config set` refused API keys. | `config set openai.apiKey` and `config set anthropic.apiKey` write to `~/.libscope/secrets.json`. | +| `llm.provider` had no default; `ask` failed until it was set. | `llm.provider` defaults to `auto`: passthrough under MCP, else OpenAI if an OpenAI key is set, else Anthropic if an Anthropic key is set, else Ollama if `llm.url` is set or the embedding provider is `ollama`. | +| An invalid value in a config file was used as is. | An invalid value is ignored with a warning, and the default (or lower layer) applies. | +| `LIBSCOPE_ALLOW_PRIVATE_URLS=false` did not override `true` in a file. | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=false` (or `0`) sets `false`. | + +### Library (SDK) code + +| Old (1.x) | New (2.0) | +| ----------------------------------------------------------- | ---------------------------------- | +| `LibScopeConfig.embedding.openaiApiKey`, `llm.openaiApiKey` | `LibScopeConfig.openai.apiKey` | +| `LibScopeConfig.llm.anthropicApiKey` | `LibScopeConfig.anthropic.apiKey` | +| `embedding.ollamaUrl`, `ollamaModel`, `openaiModel` | `embedding.url`, `embedding.model` | +| `llm.ollamaUrl` | `llm.url` | + +## Library API + +### Renamed exports + +| Old (1.x) | New (2.0) | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `removeTagFromDocument(db, documentId, tagId)` | `removeTagsFromDocument(db, documentId, tagNames)`. Takes tag names (case-insensitive) and returns the names that were removed. | + +### Error changes + +Every "not found" error is now a `NotFoundError` (`DocumentNotFoundError`, `ChunkNotFoundError` and `TopicNotFoundError` extend it). The REST API returns HTTP 404 for all of them. Check `err.code` to tell them apart. + +| Function | 1.x error | 2.0 error (`code`) | +| ---------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------ | +| `deleteLink` with an unknown link ID | `ValidationError` | `NotFoundError` (`LINK_NOT_FOUND`) | +| `getWebhook`, `deleteWebhook`, `updateWebhook` with an unknown ID | `ValidationError` | `NotFoundError` (`WEBHOOK_NOT_FOUND`) | +| `getSavedSearch`, `deleteSavedSearch`, `runSavedSearch` with an unknown name or ID | `DocumentNotFoundError` | `NotFoundError` (`SAVED_SEARCH_NOT_FOUND`) | +| `getVersion`, `rollbackToVersion` with an unknown version | `DocumentNotFoundError` | `NotFoundError` (`VERSION_NOT_FOUND`) | +| `removePack` for a pack that is not installed | `ValidationError` | `NotFoundError` (`PACK_NOT_FOUND`) | +| `getRelatedChunks` with an unknown chunk ID | `Error` | `ChunkNotFoundError` (`CHUNK_NOT_FOUND`) | + +### Package root exports + +`import ... from "libscope"` now gives only the `LibScope` class, its option and result types (`LibScopeOptions`, `RunOptions`, `LibScopeInput`, `LibScopeOutput`, `AddInput`, `SearchOutput`, `AskOutput`, `SearchResult`, `RagResult`, `DocumentView`, `Overview`, `Task`, ...), the provider interfaces (`EmbeddingProvider`, `LlmProvider`, `Chunker`) and the error classes. The other 142 runtime exports are removed. The functions in "Renamed exports" and "Error changes" above are no longer exported from the root; the `LibScope` methods that call them throw the same errors. + +Each method below is on a `LibScope` instance (`const scope = LibScope.create()`) and takes one input object. "—" means there is no replacement in the library API. + +| Removed exports (1.x) | Replacement (2.0) | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `indexDocument`, `indexFile`, `batchImport`, `indexRepository`, `parseRepoUrl`, `fetchAndConvert`, `DEFAULT_FETCH_OPTIONS` | `scope.add()` (content, file, directory, URL with optional `spider`, or repository URL) | +| `chunkContent` | `chunker` option of `LibScope.create()` to replace the chunker; otherwise — | +| `getParserForFile`, `getSupportedExtensions` | `scope.add()`; `normalizeRawInput()` from `libscope/lite` | +| `checkDuplicate`, `findDuplicates` | `scope.add({ dedup })`, `scope.admin.dedupe()` | +| `searchDocuments` | `scope.search()` | +| `searchBatch`, `BATCH_SEARCH_MAX_REQUESTS` | `Promise.all` of `scope.search()` calls | +| `askQuestion`, `askQuestionStream`, `buildContextPrompt`, `extractSources` | `scope.ask()`, `scope.askStream()`; `llm.provider: "passthrough"` returns the context prompt | +| `createLlmProvider` | `llmProvider` option, or `config: { llm: { ... } }` | +| `registerProvider`, `createEmbeddingProvider` | `provider` option (an `EmbeddingProvider` instance), or `config: { embedding: { ... } }` | +| `getDocument`, `listDocuments`, `updateDocument`, `deleteDocument` | `scope.docs.get()`, `.list()`, `.update()`, `.delete()` | +| `rateDocument`, `getDocumentRatings`, `listRatings` | `scope.docs.rate()`; the rating summary is in `scope.docs.get()`; `listRatings` — | +| `getVersionHistory`, `getVersion`, `rollbackToVersion`, `saveVersion`, `pruneVersions`, `MAX_VERSIONS_DEFAULT` | `scope.docs.history()` (includes each version's content), `scope.docs.rollback()`; versions are saved automatically | +| `createTopic`, `listTopics`, `getTopic`, `getTopicStats`, `deleteTopic`, `getDocumentsByTopic`, `renameTopic` | `scope.topics.create()`, `.list()`, `.delete()`; `scope.docs.list({ topic })`; `renameTopic` — | +| `createTag`, `listTags`, `addTagsToDocument`, `removeTagsFromDocument`, `getDocumentTags`, `getDocumentsByTag`, `suggestTags`, `deleteTag` | `scope.tags.add()` (creates tags), `.list()`, `.remove()`, `.suggest()`; `scope.docs.get()` (tags); `scope.docs.list({ tags })`; `deleteTag` — | +| `createLink`, `listLinks`, `getDocumentLinks`, `deleteLink`, `getPrerequisiteChain` | `scope.links.create()`, `.list()`, `.delete()`, `.prerequisites()` | +| `createSavedSearch`, `listSavedSearches`, `getSavedSearch`, `runSavedSearch`, `deleteSavedSearch` | `scope.searches.save()`, `.list()`, `.run()`, `.delete()` | +| `getStats` | `scope.overview()` (`stats`, `topics`, `packs`, `index`, `health`) | +| `getPopularDocuments`, `getStaleDocuments`, `getTopQueries`, `getSearchAnalytics`, `getKnowledgeGaps`, `getSearchTrends` | `scope.analytics.popular()`, `.stale()`, `.topQueries()`, `.searches()` (includes knowledge gaps and queries per day) | +| `logSearch`, `recordSearchQuery` | — (searches are recorded automatically) | +| `listAvailablePacks`, `listInstalledPacks`, `installPack`, `removePack`, `createPack` | `scope.packs.list({ available })`, `.install()`, `.remove()`, `.create()` | +| `reindex` | `scope.admin.reindex()` | +| `exportKnowledgeBase`, `importFromBackup` | `scope.admin.backup()`, `scope.admin.restore()` | +| `pruneExpiredDocuments` | `scope.admin.pruneExpired()` | +| `bulkDelete`, `bulkRetag`, `bulkMove`, `resolveSelector` | `scope.admin.bulkDelete()`, `.bulkRetag()`, `.bulkMove()` (`dryRun: true` lists the matches) | +| `createWebhook`, `listWebhooks`, `getWebhook`, `deleteWebhook`, `updateWebhook`, `signPayload`, `buildPayload`, `fireWebhooks`, `redactWebhook`, `WEBHOOK_EVENTS` | `scope.webhooks.create()`, `.list()`, `.delete()`, `.test()`; `updateWebhook`: delete and create; the others are internal — | +| `syncNotion`, `syncSlack`, `syncConfluence`, `syncOneNote`, `syncObsidianVault`, `syncDocSite` | Save the connection with the CLI (`libscope connect `), then `scope.connectors.sync({ name })` or `sync({ all: true })` | +| `disconnectNotion`, `disconnectSlack`, `disconnectConfluence`, `disconnectOneNote`, `disconnectVault`, `disconnectDocSite`, `deleteConnectorDocuments` | `scope.connectors.disconnect({ name })` | +| `saveConnectorConfig`, `loadConnectorConfig`, `saveNamedConnectorConfig`, `loadNamedConnectorConfig`, `hasNamedConnectorConfig`, `saveDbConnectorConfig`, `loadDbConnectorConfig`, `deleteDbConnectorConfig` | CLI `libscope connect `; `scope.connectors.list()` shows saved connections (secrets masked) | +| `authenticateDeviceCode`, `refreshAccessToken` | CLI `libscope connect onenote` | +| `convertNotionBlocks`, `convertSlackMrkdwn`, `convertConfluenceStorage`, `convertOneNoteHtml`, `parseObsidianMarkdown`, `detectDocSiteType`, `extractDocLinks`, `extractDocTitle`, `extractMainContent`, `extractElementByPattern`, `extractSitemapUrls`, `normalizeDocUrl` | — (connector internals) | +| `ConnectorScheduler`, `loadScheduleEntries` | CLI `libscope connect --schedule ` | +| `createWorkspace`, `deleteWorkspace`, `listWorkspaces`, `getWorkspacePath`, `getWorkspacesDir`, `getActiveWorkspace`, `setActiveWorkspace`, `DEFAULT_WORKSPACE` | `workspace` option of `LibScope.create()`; CLI `libscope workspace` | +| `FileWatcher`, `DEFAULT_WATCH_EXTENSIONS` | CLI `libscope add --watch` | +| `buildKnowledgeGraph`, `detectClusters` | `scope.links.graph()` (`detectClusters` —) | +| `startApiServer` | CLI `libscope serve api` | + +The removed type exports follow their functions. Use `LibScopeInput<"group", "method">` and `LibScopeOutput<"group", "method">` for the input and result of any namespace method. + +### `LibScope` class + +`LibScope.create()` now uses the same startup as the CLI and MCP server. Its methods call the operations, so inputs and results match the CLI, MCP and REST API. + +| 1.x | 2.0 | +| ------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LibScope.create({ config: Partial })` | `config` is per-section overrides (`ConfigOverrides`). New options: `db`, `useConfigFile`, `provider`, `llmProvider`, `chunker`. | +| `scope.index({ title, content, ... })` → `{ id, chunkCount }` | `scope.add({ title, content, ... })` → `{ kind, documents: [{ documentId, title, chunkCount, source }], errors, skipped }`. `topicId` is now `topic` (ID or name). | +| `scope.indexFile(path, options)` | `scope.add(path)` or `scope.add({ source: path, ... })` | +| `scope.search(query, options)` → `{ results, totalCount }` | `scope.search(query)` or `scope.search({ query, ...filters })` → `{ items, total, limit, offset }`. The `source` filter is now `sourceType`. | +| `scope.searchBatch(requests)` | `Promise.all(requests.map((r) => scope.search(r)))` | +| `scope.ask(question, options)` → `RagResult` | `scope.ask(question)` or `scope.ask({ question, ...filters, topK, systemPrompt })` → `{ mode: "answer", ...RagResult }` or, with `llm.provider: "passthrough"`, `{ mode: "context", contextPrompt, sources }` | +| `scope.askStream(question, options)` | `scope.askStream(question)` or `scope.askStream({ question, ... })`. Same events. | +| `scope.stats()` | `(await scope.overview()).stats` | +| `scope.list(options)` → `Document[]` | `scope.docs.list({ topic, library, version, sourceType, tags, limit, offset })` → `{ items, total, limit, offset }`. Items have `documentId` and `contentLength`, not `id` and `content`. `dateFrom`/`dateTo` are removed. | +| `scope.get(id)` → `Document` | `scope.docs.get({ documentId })` → `{ document, content, contentLength, offset, nextOffset, tags, links, ratings }` | +| `scope.delete(id)` | `scope.docs.delete({ documentId })` | +| Synchronous `stats()`, `list()`, `get()`, `delete()` | Every method returns a promise. | + +### `libscope/lite` + +The `LibScopeLite` class is removed. `createLite(options)` returns a `LibScope` with the same methods and results as `LibScope.create()`. It does not read config files or `LIBSCOPE_*` config variables. + +| 1.x | 2.0 | +| --------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `new LibScopeLite({ dbPath, provider, llmProvider })` | `createLite({ dbPath, provider, llmProvider })`. `dbPath` (or `db`) is required; the old default `~/.libscope/lite.db` is removed. | +| `new LibScopeLite({ db })` skipped migrations | `createLite({ db })` migrates the database and creates the vector table. | +| `model` option (unused) | Removed. Use `config: { embedding: { model } }` or `provider`. | +| `lite.index(docs)`, `lite.indexBatch(docs, { concurrency })` | `scope.add({ title, content, ... })` per document (run them in parallel if you want). | +| `LiteDoc.language` (tree-sitter chunking) | `createLite({ chunker: createCodeChunker() })`. The language comes from the title or source extension, or `createCodeChunker({ language })`. | +| `lite.indexRaw(input)` → `string[]` | `const { title, content } = await normalizeRawInput(input)`, then `scope.add({ title, content })`. Code is no longer split into "part N" documents; use a chunker. | +| `lite.search(query, opts)` → `LiteSearchResult[]` with `docId` | `scope.search({ query, ...opts })` → `{ items, total, limit, offset }`; items have `documentId`. | +| `lite.getContext(question, opts)` → `string` | `createLite({ ..., config: { llm: { provider: "passthrough" } } })`, then `(await scope.ask(question)).contextPrompt`. | +| `lite.ask(question, opts)` → `string` | `scope.ask({ question, topK, systemPrompt })` → `{ mode: "answer", answer, sources, model }`. The per-call `llmProvider` option is removed; pass it to `createLite`. Without an LLM, `ConfigError` (was `Error`). | +| `lite.askStream(question)` → tokens (`string`) | `scope.askStream(question)` → `{ token }` events, then `{ done: true, sources, model }`. A provider without `completeStream` now works (one token). | +| `lite.rate(docId, score)` | `scope.docs.rate({ documentId, rating })` | +| `lite.deleteByLibrary(library)` | `scope.admin.bulkDelete({ library })` (up to 1000 per call; repeat until `affected` is 0) | +| Types `LiteOptions`, `LiteDoc`, `LiteSearchOptions`, `LiteSearchResult`, `LiteContextOptions`, `LiteAskOptions` | `LiteOptions` (new shape), `AddInput`, `SearchInput`, `SearchResult`, `AskInput` | +| `NormalizedInput.chunks` | Removed. `normalizeRawInput` returns `{ title, content }`. | + +### Package `exports` + +`libscope` and `libscope/lite` list `types` first and use `default` (no separate `import`/`require` entries). + +## MCP + +The MCP server now has 11 core tools and an optional admin toolset. Old tool names do not work in 2.0. Update prompts, agent configs and allow-lists that name MCP tools. + +### Server + +| 1.x | 2.0 | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| `import "libscope/mcp"` started a stdio server. | Importing `libscope/mcp` starts nothing. Call `runStdioServer()`, or `createMcpServer()` and connect your own transport. | +| `npm run serve` ran `dist/mcp/server.js`. | `npm run serve` runs `dist/mcp/main.js`. `node dist/mcp/server.js` exits without serving. | +| The server reported version `0.1.0`. | The server reports the package version. | +| Every tool was always registered. | `ask` is registered only with passthrough or a configured LLM. Admin tools are registered only with `LIBSCOPE_MCP_TOOLSETS=admin` (or `all`). | +| Most tools returned markdown or pretty-printed JSON. | Tools return compact text. Every document, chunk, link and task line includes its ID. | +| Long-running tools accepted `async` and polled `get-task`. | Same `async: true`, then `task {"action": "status", "taskId"}`. A finished task shows the same output as the tool run without `async`. | + +### Renamed and merged tools + +| 1.x tool | 2.0 tool | Parameter changes | +| ------------------------------------------------------------------------------------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `search-docs` | `search` | `source` -> `sourceType` (enum). Added `tags`, `relatedTo`, `diversity`. `limit` max 100. `maxChunksPerDocument` minimum is 1. | +| `get-related` | `search` with `relatedTo` | `chunkId` -> `relatedTo` (a chunk ID or a document ID). `minScore` and `includeLinkedDocuments` are removed. Use `version`, `sourceType`, `minRating` as filters. | +| `ask-question` | `ask` | Added `version`, `sourceType`, `tags`, `minRating`, `systemPrompt`. | +| `health-check`, `list-topics` | `overview` | `list-topics` `parentId` is removed; `overview` lists every topic with its document count. | +| `list-packs` | `overview` (installed packs), or admin `list-packs` | Admin `list-packs` keeps `available`; `registryUrl` is replaced by `registry` (a git registry name). | +| `get-document` | `get-document` | Added `offset` and `maxLength` for paging. The output includes tags, links (with `linkId`) and ratings. | +| `get-document-links` | `get-document` | Links are part of the document output. | +| `list-documents` | `list-documents` | Added `version`, `tags`, `offset`. `limit` max 1000. The output shows the total. | +| `submit-document` | `submit-document` | Added `tags`, `expiresAt`, `dedup`, `dryRun` and repository fields (`branch`, `paths`, `extensions`). A GitHub/GitLab repository URL indexes the repository. `topic` must be an existing topic ID or name. | +| `update-document` | `update-document` | `topicId` -> `topic` (ID or name). Added `tags` (replaces the document's tags). | +| `link-documents` | `link-documents` with `action: "create"` | `sourceId` -> `documentId`, `targetId` -> `targetDocumentId`. | +| `delete-link` | `link-documents` with `action: "delete"` | `linkId` unchanged. | +| `get-task` | `task` with `action: "status"` | `taskId` unchanged. An unknown task is an error. | +| `cancel-task` | `task` with `action: "cancel"` | `taskId` unchanged. An unknown task is an error. | +| — | `task` with `action: "list"` | New. | +| `install-pack` (always on) | `install-pack` (admin toolset) | `nameOrPath` -> `pack`. Added `batchSize`, `concurrency`, `resumeFrom`. | +| `reindex-documents` (always on) | `reindex-documents` (admin toolset) | Added `rebuild`. | +| `sync-slack`, `sync-notion`, `sync-confluence`, `sync-onenote`, `sync-obsidian-vault` | `sync` (admin toolset) | `{ "name": "" }` or `{ "all": true }`. Credentials and connector settings are no longer tool parameters: save them with `libscope connect `. | + +`delete-document` and `rate-document` keep their names and parameters. + +### Removed tools + +These operations are available from the CLI, the REST API and the SDK, not over MCP: + +| 1.x tool | Operation (REST path under `/api/v1`) | +| ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| `save-search`, `list-saved-searches`, `run-saved-search`, `delete-saved-search` | `save-search`, `list-saved-searches`, `run-saved-search`, `delete-saved-search` (`/searches`) | +| `create-webhook`, `list-webhooks`, `delete-webhook` | `create-webhook`, `list-webhooks`, `delete-webhook` (`/webhooks`) | +| `search-analytics` | `search-analytics` (`/analytics/searches`) | +| `suggest-tags` | `suggest-tags` (`/documents/:documentId/suggested-tags`) | + +## REST API + +Every `/api/v1` route now runs one operation and is generated from it. The [REST API reference](reference/rest-api.md) lists all routes. The OpenAPI document at `GET /openapi.json` is generated from the same routes (OpenAPI 3.1.0, was a hand-written 3.0.3 document that listed 14 routes). + +### Routes + +| Old route (1.x) | New route (2.0) | +| --------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `GET /api/v1/search?q=&source=&tag=` | `GET /api/v1/search?query=&sourceType=&tags=` (also `library`, `version`, `minRating`, `relatedTo`) | +| `POST /api/v1/batch-search` | Removed. Send one `GET /api/v1/search` per query. | +| `POST /api/v1/documents/url` | `POST /api/v1/documents` with `url` (and `spider: true` to crawl) | +| `POST /api/v1/index/repos/:repoSlug` | Removed with the REST repo indexer. `POST /api/v1/documents` with `source: ""`. | +| `GET /api/v1/index/jobs/:jobId` | `GET /api/v1/tasks/:taskId` | +| `GET /api/v1/stats` | `GET /api/v1/overview` (`data.stats`) | +| `GET /api/v1/connectors/status` | `GET /api/v1/connections` (`lastRun` of each connection). The `history=true` sync history is removed. | +| `GET /api/v1/connectors/schedules` | `GET /api/v1/connections` (`schedule` of each connection) | +| `GET /api/v1/documents/:id/links` | `GET /api/v1/links?documentId=:id` | +| `GET /api/v1/documents/:id/suggest-tags` | `GET /api/v1/documents/:id/suggested-tags` | +| `POST /api/v1/bulk/:operation` with any other name | Only `delete`, `retag` and `move` exist; other names are `404`. | +| `POST /api/v1/ask` with `Accept: text/event-stream` | Removed (no streaming). `POST /api/v1/ask` returns the whole answer. | + +These routes did not change their path: `GET /openapi.json`, `GET /api/v1/health`, `GET` and `POST /api/v1/documents`, `GET`, `PATCH` and `DELETE /api/v1/documents/:id`, `POST /api/v1/documents/:id/tags`, `POST /api/v1/documents/:id/links`, `DELETE /api/v1/links/:id`, `POST /api/v1/ask`, `GET` and `POST /api/v1/topics`, `GET /api/v1/tags`, `GET /api/v1/analytics/searches`, the saved-search routes, `POST /api/v1/bulk/{delete,retag,move}` and the webhook routes. + +New routes: `DELETE /api/v1/documents/:id/tags`, `POST /api/v1/documents/:id/ratings`, `GET /api/v1/documents/:id/versions`, `POST /api/v1/documents/:id/rollback`, `GET /api/v1/documents/:id/prerequisites`, `GET /api/v1/links`, `GET /api/v1/graph`, `DELETE /api/v1/topics/:topic`, `GET` and `POST /api/v1/packs`, `DELETE /api/v1/packs/:pack`, `GET /api/v1/registries`, `GET /api/v1/registries/search`, `GET /api/v1/connections`, `DELETE /api/v1/connections/:name`, `POST /api/v1/sync`, `GET /api/v1/overview`, `POST /api/v1/admin/reindex`, `GET /api/v1/admin/duplicates`, `POST /api/v1/admin/prune-expired`, `GET /api/v1/analytics/{popular,stale,top-queries}`, `GET /api/v1/tasks`, `GET /api/v1/tasks/:taskId` and `POST /api/v1/tasks/:taskId/cancel`. + +### Request fields + +| Route | Old field (1.x) | New field (2.0) | +| ------------------------------------- | ------------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `POST /api/v1/documents` | `source` (the source type) | `sourceType`. `source` now names what to add (URL or repository). | +| `POST /api/v1/documents` | `topic` (topic ID) | `topic` (topic ID or name; the topic must exist) | +| `PATCH /api/v1/documents/:id` | `topicId` | `topic` (ID or name; `null` clears it). New: `tags` replaces the tags. | +| `POST /api/v1/documents/:id/links` | `targetId` | `targetDocumentId` | +| `POST /api/v1/searches` | `filters: { topic, library, ... }` | The filter fields at the top level: `topic`, `library`, `version`, `sourceType`, `tags`, `minRating`, `limit` | +| `POST /api/v1/bulk/*` | `selector: { topicId, library, ... }` | The selector fields at the top level: `topic`, `library`, `sourceType`, `tags`, `dateFrom`, `dateTo` | +| `POST /api/v1/bulk/move` | `targetTopicId` | `targetTopic` (ID or name) | +| `GET` routes with `limit` or `offset` | Out-of-range values were clamped | Out-of-range values are rejected with `400` | + +### Responses + +| 1.x | 2.0 | +| -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `POST /api/v1/documents` indexed the document and answered `201` with it. | It starts a task and answers `202` with `taskId`. Poll `GET /api/v1/tasks/:taskId`; `result` (a JSON string) lists the added documents with their `documentId`. | +| Create routes answered `201`; `DELETE` routes answered `204` with no body. | Every synchronous route answers `200` with `{ data, meta }`. `DELETE` returns, for example, `{ "documentId": "...", "deleted": true }`. | +| `GET /api/v1/search` returned `{ results, totalCount }`. | `{ items, total, limit, offset }` | +| `GET /api/v1/documents` returned an array of documents with their content. | `{ items, total, limit, offset }`. Items have `documentId` and `contentLength`, not `id` and `content`. | +| `GET /api/v1/documents/:id` returned the document row. | `{ document, content, contentLength, offset, nextOffset, tags, links, ratings }`. Use `offset` and `maxLength` to page the content. | +| `GET /api/v1/topics`, `/tags`, `/webhooks`, `/searches` returned arrays. | `{ items, ... }` | +| `GET /api/v1/health` returned `{ status, docCount, dbSize }`. | `{ status: "ok" }`. Counts and index health are in `GET /api/v1/overview`. | +| Links had `id`. | Links have `linkId`. | +| Not-found errors had the code `NOT_FOUND`. | The code names the resource: `DOCUMENT_NOT_FOUND`, `TOPIC_NOT_FOUND`, `LINK_NOT_FOUND`, `TASK_NOT_FOUND`, ... Unknown routes keep `NOT_FOUND`. | +| A body over 1 MB answered `500`. | `413` with the code `PAYLOAD_TOO_LARGE` | +| `ask` without an LLM answered `500` with a generic message. | `500` with the code `CONFIG_ERROR` and the setting to change | + +### CORS and browser requests + +| 1.x | 2.0 | +| ----------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `corsOrigins: ["*"]` allowed every method from every origin. | `"*"` allows `GET` only. Name an origin in `corsOrigins` to let it write. | +| A page on any origin could send a write request without a preflight (for example, a form POST). | Write requests from an origin that is not listed, and is not the server's own origin, get `403` (`FORBIDDEN_ORIGIN`). | + +### REST repo indexer (removed) + +The REST repo indexer is removed: the routes `POST /api/v1/index/repos/:repoSlug` and `GET /api/v1/index/jobs/:jobId`, the `LIBSCOPE_REPOS_CONFIG` environment variable and its repos JSON file. It indexed each repository into its own database (`~/.libscope/repos/.db`) that no other part of LibScope could search, and it always used the local embedding model. + +Add a repository to the workspace database instead: + +```bash +curl -X POST http://localhost:3378/api/v1/documents \ + -H "Content-Type: application/json" \ + -d '{ "source": "https://github.com/org/repo", "branch": "main", "paths": ["docs"] }' +``` + +LibScope 2.0 does not read or delete the files in `~/.libscope/repos/`. Delete them yourself when you no longer need them. + +### Web dashboard server + +The dashboard URLs (`/`, `/graph`, `/api/stats`, `/api/topics`, `/api/documents`, `/api/documents/:id`, `/api/search`, `/api/graph`) keep their paths and response shapes, with these changes: + +| 1.x | 2.0 | +| ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | +| Every response had `Access-Control-Allow-Origin: *`, also for `DELETE`. | No CORS header by default (same origin only). Cross-origin `DELETE` gets `403`. | +| Errors were `{ "error": "" }`. | `{ "error": { "code", "message" } }` | +| `GET /api/documents` items had `content`. | Items have `contentLength` instead. | +| Out-of-range `limit` values were clamped. | Out-of-range values are rejected with `400` (`limit` 1-1000 for documents, 1-100 for search). | + +### Library code + +| Old (1.x) | New (2.0) | +| ------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `startApiServer(db, provider, options)` | `startApiServer({ db, provider, config }, options)`. Pass the result of `bootstrap()`. The returned `port` is the bound port. | +| `startWebServer(db, provider, options)` | `startWebServer({ db, provider, config }, options)` | +| `handleRequest(req, res, db, provider)` (`src/api/routes.ts`) | `handleRequest(req, res, ctx)` with an `OperationContext` from `createOperationContext` | +| `OPENAPI_SPEC` (`src/api/openapi.ts`) | `buildOpenApiSpec(API_ROUTES)` | +| `handleGraphRequest` (`src/web/graph-api.ts`) | The `graph` operation (`graphOperation`) | + +## CLI + +The CLI has one command per task, and every command calls the same operation as the MCP server, REST API and SDK. The 1.x commands below are removed without aliases. See the [CLI reference](reference/cli.md). + +### Removed and renamed commands + +| 1.x command | 2.0 command | +| ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| `init` | Not needed: the database is created on first use. `doctor` shows the setup; `doctor --fix` creates the database. | +| `add ` | `add ` (several sources at once) | +| `import [--extensions .md,.txt]` | `add [--include "*.md,*.txt"]` | +| `import-batch [--filter ] [--dry-run]` | `add [--include ] [--dry-run]` | +| `add-repo [--path ]` | `add [--path ]` (`--branch`, `--extensions`, `--token` are unchanged) | +| `watch ` | `add --watch` | +| `search ` (query required) | `search [query]` | +| `repl` | `search` with no query (on a terminal) | +| `related ` | `search --related ` | +| `search --save ` | `searches save [filters]` | +| `ratings show ` | `docs show ` (shows the rating summary) | +| `link ` | `docs link --type ` | +| `links ` | `docs links [documentId]` | +| `unlink ` | `docs unlink ` | +| `prereqs ` | `docs prereqs ` | +| `tag add ` | `docs tag ` | +| `tag remove ` | `docs untag ` | +| `tag list` | `tags list` | +| `tag suggest ` | `docs suggest-tags ` | +| `stats`, `stats overview`, `stats popular`, `stats stale`, `stats queries`, `stats search-analytics` | `admin stats` (one report) | +| `reindex` | `admin reindex` | +| `dedupe` | `admin dedupe` | +| `export ` | `admin backup ` | +| `import-backup ` | `admin restore ` | +| `serve --api` | `serve api` | +| `serve --dashboard` | `serve dashboard` | +| `connect notion`, `slack`, `confluence`, `onenote`, `obsidian` with `--sync` | `sync ` (or `sync --all`) | +| `connect confluence --url --type server` | `connect confluence --server` | +| `disconnect notion`, `slack`, `confluence`, `onenote` `[--name ]` | `disconnect ` (the connection name; the default name is the type) | +| `disconnect obsidian ` | `disconnect ` (the vault path comes from the saved connection) | +| `schedule set ` | `connect --name --schedule --no-sync` | +| `schedule remove ` | `connect --name --schedule off --no-sync` | +| `schedule list` | `connections` | +| `update` (self-update) | `npm install -g libscope@latest` | + +New in 2.0: `doctor`, `docs rate`, `topics delete`, `admin prune`, `connect docs `, `disconnect --keep-documents`, `connections`. + +### Removed and renamed options + +| 1.x option | 2.0 option | +| ------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------- | +| `search --source ` | `--source-type ` (the same on every command that filters documents) | +| `search --limit ` (default 5) | `-n, --limit ` (default 10) | +| `search --max-chunks-per-doc ` | `search --max-per-doc ` | +| `ask --top-k ` | `ask -n ` | +| `related --min-score ` | Removed | +| `add --exclude-patterns ` (crawl) | `add --exclude-urls ` (comma-separated) | +| `bulk retag --add-tags`, `--remove-tags` | `bulk retag --add`, `--remove` | +| `link --type` defaulted to `related` | `docs link --type ` is required | +| `stats stale --days`, `stats search-analytics --days` | `admin stats --days ` | +| `connect slack --exclude`, `connect confluence --exclude-spaces` | `connect --exclude ` for every type | +| `connect onenote` read `ONENOTE_CLIENT_ID` and `ONENOTE_TENANT_ID` | `connect onenote --client-id --tenant-id ` | +| `connect notion` read `NOTION_TOKEN`; `connect confluence` read `CONFLUENCE_URL`, `CONFLUENCE_EMAIL`, `CONFLUENCE_TOKEN` | Give `--token`, the URL and `--email` on the command line | +| `-v, --verbose` only enabled debug logs | `-v, --verbose` also prints the stack trace when a command fails | + +### Behavior changes + +| 1.x | 2.0 | +| ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| Errors were printed in several formats, some with a stack trace. | Every error prints `✗ ` and a next step on stderr, and exits with code 1. Add `--verbose` for the stack trace. | +| No machine-readable output. | The global `--json` option prints the operation result as JSON. | +| Some results did not show IDs. | Every result shows document IDs; search results also show chunk IDs. | +| `add ` stored no URL, so adding a file twice made two documents. | The file's absolute path is stored as its URL. Adding a changed file again replaces the old version. | +| `--topic` took a topic ID. | `--topic` takes a topic ID or name on every command. | +| `connect ` always started from scratch. | `connect ` creates or updates a saved connection (only the given settings change) and syncs it. | +| `import-backup` did not ask for confirmation. | `admin restore` asks; add `-y` to skip the question. | +| Import progress was printed on stdout. | Progress is one line on stderr, only on a terminal. | + +## Packs and registries + +LibScope 1.x had two pack registry systems: a URL registry (one JSON file, by default `https://raw.githubusercontent.com/libscope/packs/main/registry.json`) and git registries (`libscope registry add`). LibScope 2.0 keeps only the git registries. `--registry ` (`registry` in the SDK, REST and MCP) always means the name of a configured git registry. See the [Pack Registries guide](guide/pack-registries.md). + +### Removed: the URL registry + +| 1.x | 2.0 | +| ---------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| `pack install --registry ` | `registry add `, then `pack install ` | +| `pack list --available --registry ` | `pack list --available [--registry ]` (packs in the configured git registries) | +| `pack install ` without git registries fetched `.json` from the URL registry | Fails with "no pack registries are configured"; add a registry first | +| `install-pack` and `list-packs` input `registryUrl` (MCP, REST, SDK) | `registry` (a registry name) | +| `listAvailablePacks(registryUrl)`, `PackInfo` (`src/core/packs.ts`) | `listRegistryPacks(registryName?)` (`src/registry/search.ts`), `RegistryPack` | +| `installPack(db, provider, nameOrPath, { registryUrl })` | `installPack(db, provider, packFile)`; resolve names with `resolveRegistryPack(spec, registry?)` | + +### Removed and renamed options + +| 1.x option | 2.0 | +| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `pack install --from-registry ` | `pack install --registry ` | +| `pack install --pack-version ` | `pack install @` | +| `pack install -y` (with several registries, use the highest priority) | Removed. A pack in several registries is an error that names them; use `--registry ` | +| Interactive registry choice in `pack install` | Removed (same error) | +| `registry add --priority `, config field `registries[].priority` | Removed (no priority order; the field is ignored) | +| `registry add --sync-interval `, config field `registries[].syncInterval` | Removed. It never triggered a sync; run `registry sync` | +| `registry search -r `, `registry publish -r `, `registry unpublish -r ` | `--registry ` | +| `registry unpublish --pack-version ` | `registry unpublish @` | + +### Behavior changes + +| 1.x | 2.0 | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| MCP `install-pack` accepted a local `.json`/`.json.gz` path. | Local pack files are installed only from the CLI and the SDK (MCP and REST refuse them). MCP and REST install by name. | +| A pack in several registries was resolved by priority or a prompt. | It is a `ValidationError` that names the registries. | +| An unknown registry, pack or version was a `ValidationError` or `null`. | It is a `NotFoundError` (404 over REST). | +| `registry add` accepted only `https://`, `ssh://` and `git@host:path`. | `file:///` URLs (a repository on a local or shared disk) are also accepted. | +| `registry` commands exited with their own messages. | They call the registry operations and print errors like every other command. | +| `pack list --available` read the URL registry over the network. | It reads the local copies of the git registries. Only `registry add`, `sync`, `publish` and `unpublish` use the network. | + +### New operations + +`list-registries`, `add-registry`, `remove-registry`, `sync-registries`, `search-registries`, `create-registry`, `publish-pack` and `unpublish-pack`. In the SDK they are `scope.registries.list()`, `.add()`, `.remove()`, `.sync()`, `.search()`, `.create()`, `.publish()` and `.unpublish()`. Over REST, only the read-only ones have routes: `GET /api/v1/registries` and `GET /api/v1/registries/search`. They are not MCP tools. + +### Library code + +| 1.x | 2.0 | +| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| `findPackInRegistries`, `resolvePackFromRegistries` (`src/registry/resolve.ts`) | `findRegistryPack(spec, registry?)`, `resolveRegistryPack(spec, registry?)` | +| `RegistryConflict`, `ConflictResolution`, `RegistryConfigBlock` types | Removed | +| `syncRegistryByName`, `syncStaleRegistries`, `getRegistryIndex`, `isRegistryStale` (`src/registry/sync.ts`) | `syncRegistry(requireRegistry(name))`, `syncAllRegistries()`, `listRegistryPacks()` | +| `checkGitAvailable` (`src/registry/git.ts`) | Removed: a missing git binary is a `ConfigError` from any git call | +| `REGISTRIES_DIR` constant | `getRegistriesDir()` (follows the current home directory) | +| `RegistrySearchResult` `{ registryName, pack, score }` | `{ ...pack, registry, score }` | +| `RegistrySyncStatus.registryName`, status `"syncing"` | `registry`; statuses `success`, `offline`, `error`; new field `packs` | +| `ResolvedPack` `{ registryName, registryUrl, packName, version, dataPath }` | `{ registry, name, version, dataPath }` | + +## Client SDKs + +The Python (`pylibscope`) and Go (`sdk/go`) clients now call the 2.0 `/api/v1` routes. Methods that add documents or sync connections start a background task on the server and return a task at once. Wait for it with `wait_for_task` (Python) or `WaitForTask` (Go). Both poll `GET /api/v1/tasks/:taskId`. Both clients accept an API key (`api_key=` / `WithAPIKey`), sent as `Authorization: Bearer `. + +### Python (`pylibscope`) + +| 1.x | 2.0 | +| --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `add_document(url, ...)` returned a `Document` | `add_url(url, *, spider, max_pages, ...)` returns a `Task`. Get the documents from `wait_for_task(task.id).result["documents"]`. | +| `add_text(title, content, ...)` returned a `Document` | `add_text(...)` returns a `Task` | +| `search(query, *, limit, topic, tags, min_score)` | `search(query, *, limit, offset, topic, library, version, source_type, tags, min_rating)` returns `Page[SearchHit]`. `min_score` is removed. Every tag in `tags` is sent (1.x sent only the first). | +| `SearchResult` (`results`, `total_count`) | `Page[SearchHit]` (`items`, `total`, `limit`, `offset`) | +| `get_document(doc_id)` returned a `Document` | `get_document(document_id)` returns a `DocumentView` (`document`, `content`, `tags`, `links`, `ratings`) | +| `list_documents(...)` returned `List[Document]` | `list_documents(...)` returns `Page[Document]` | +| `Document.id` | `Document.document_id` | +| `get_analytics()` and `Analytics` | `overview()` and `Overview` (`overview().stats.total_documents`) | +| `create_topic(name, *, parent_id)` | `create_topic(name, *, parent, description)` | +| `add_tags` and `remove_tags` returned `None` | Both return the document's tags. `remove_tags` now removes the tags (1.x sent a body that the server ignored). | +| `list_tags()` returned `List[str]` | `list_tags()` returns `List[Tag]` (`name`, `document_count`) | +| `get_graph(*, min_similarity)` | `get_graph(*, topic, tag, threshold, max_nodes)` | +| `ask(question, *, topic)` returned `AskResult(answer, sources)` | `ask(question, *, top_k, min_rating, topic, library, version, source_type, tags)` returns `AskResult` (`mode`, `answer` or `context_prompt`, `sources`) | +| `sync_connector(connector, **config)` and `SyncResult` | `sync(name)` and `sync_all()` return a `Task`. Save connections with `libscope connect `. | +| `pylibscope.connectors.build_connector_config` | Removed | +| — | New: `get_task`, `cancel_task`, `wait_for_task`, `TaskFailedError`, `Tag`, `Overview`, `DocumentView`, `Page` | + +`AsyncLibscopeClient.wait_for_task(task_id, *, interval)` has no `timeout` parameter. Set the limit around the call: `await asyncio.wait_for(client.wait_for_task(task.id), timeout=300)`, or `async with asyncio.timeout(300):` on Python 3.11 and later. The synchronous `LibscopeClient.wait_for_task` keeps `timeout`. + +### Go (`github.com/RobertLD/libscope/sdk/go`) + +| 1.x | 2.0 | +| ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `AddDocument(ctx, url, opts...)` returned `*Document` | `AddURL(ctx, url, params...)` returns `*Task`. Call `WaitForTask`, then `Task.DecodeResult` into an `IngestResult`. | +| `AddText(ctx, title, content, opts...)` returned `*Document` | `AddText(ctx, title, content, params...)` returns `*Task` | +| `Search(...)` returned `*SearchResult` (`Results`, `Total`, `Query`) | `Search(...)` returns `*Page[SearchHit]` (`Items`, `Total`, `Limit`, `Offset`) | +| `SearchHit.Document`, `SearchHit.ChunkText` | `SearchHit.DocumentID`, `ChunkID`, `Title`, `Content` and the other document fields | +| `GetDocument` returned `*Document` | `GetDocument` returns `*DocumentView` | +| `ListDocuments` returned `[]Document` | `ListDocuments` returns `*Page[Document]` | +| `Document.ID`, `Document.Source` | `Document.DocumentID`, `Document.SourceType` | +| `GetStats(ctx)` and `Stats{TotalDocuments, DatabaseSize}` | `Overview(ctx)`. Counts are in `Overview.Stats` (`TotalDocuments`, `TotalChunks`, `TotalTopics`, `DatabaseSizeBytes`). | +| `HealthStatus.DocCount`, `HealthStatus.DBSize` | Removed. `Health` returns only `Status`. | +| `AddTagsToDocument(ctx, docID, tags)` returned `[]Tag` | `AddTags(ctx, id, tags)` returns the document's tags (`[]string`). New: `RemoveTags`. | +| Option types `SearchOption`, `AddDocumentOption`, `AddTextOption`, `ListOption`, `CreateTopicOption` | One `Param` type for all methods | +| `WithDocumentTopic`, `WithTextTopic`, `WithListTopic` | `WithTopic` | +| `WithDocumentTags`, `WithTextTags` | `WithTags` | +| `WithListLimit` | `WithLimit` | +| `WithTextSource` | `WithSourceType` | +| `WithParentID` | `WithParent` | +| `WithMinScore` (filtered on the client) | Removed. `WithMinRating` filters by document rating on the server. | +| `WithTags(...)` sent only the first tag | `WithTags(...)` sends every tag | +| — | New: `Ask`, `GetGraph`, `Sync`, `SyncAll`, `GetTask`, `CancelTask`, `WaitForTask`, `WithAPIKey`, `TaskError`. `errors.Is(err, ErrNotFound)` now works. | + +## Saved searches + +Saved search filters use the field names of the `search` operation: `sourceType` (was `source`), `topic`, `library`, `version`, `tags`, `minRating` and `limit`. Database migration 19 (schema version 19) renames `source` to `sourceType` in existing saved searches. A saved search runs the `search` operation with its stored filters. That operation ignores stored keys that it does not accept, for example `dateFrom` and `dateTo` (the 1.x REST API could store them). + +## Docker image + +| 1.x | 2.0 | +| -------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | +| `CMD ["serve"]` started the MCP server on stdio | `CMD ["serve", "api", "--host", "0.0.0.0", "--port", "3378"]` starts the REST API | +| Exposed ports 3420 and 3421 | Exposes port 3378 (REST API). docker-compose maps `3378:3378` and passes `LIBSCOPE_API_KEY`. | +| `LIBSCOPE_DATA_DIR=/data` (LibScope did not read it, so data was not stored in the volume) | `HOME=/data`. Config, secrets, connections and workspace databases are in `/data/.libscope`. | +| The health check called `GET /api/v1/stats` | The health check calls `GET /api/v1/health` | +| `node:22-alpine` base image: the local embedding model failed to load (onnxruntime-node needs glibc), so adding documents failed with the default config | `node:22-bookworm-slim` base image | diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 2919a2b1..b21c7c38 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1,388 +1,821 @@ # CLI Commands -## Core Commands +Every command is a thin wrapper over one LibScope operation, so the CLI, the MCP server, the REST API and the SDK use the same parameters and defaults. Upgrading from 1.x? See [Migrating to LibScope 2.0](/migration-v2#cli). -| Command | Description | -| ----------------------------------- | ---------------------------------------------------------------------- | -| `libscope init` | Initialize the database | -| `libscope add ` | Index a document from a file or URL | -| `libscope import ` | Bulk import files from a directory | -| `libscope import-batch ` | Batch import with parallel processing | -| `libscope search ` | Semantic search | -| `libscope ask ` | RAG question-answering | -| `libscope repl` | Interactive search REPL | -| `libscope serve` | Start MCP server, REST API (`--api`), or web dashboard (`--dashboard`) | +The usage lines and option tables on this page are generated from the CLI code (`npm run docs:gen`). `libscope --help` prints the same text. -### `libscope add` +| Command | What it does | +| ------------------------------------------------------------- | -------------------------------------------------------------- | +| [`add`](#libscope-add) | Add files, directories, URLs (optionally crawled) or repos | +| [`search`](#libscope-search) | Search by meaning and keywords; interactive with no query | +| [`ask`](#libscope-ask) | Answer a question with the configured LLM | +| [`docs`](#libscope-docs) | Show, change, link, tag, rate and version single documents | +| [`topics`, `tags`](#topics-and-tags) | Manage topics; list tags | +| [`searches`](#libscope-searches) | Save and re-run searches | +| [`bulk`](#libscope-bulk) | Delete, retag or move every document that matches filters | +| [`connect`, `sync`, `disconnect`, `connections`](#connectors) | Notion, Slack, Confluence, Obsidian, OneNote and doc sites | +| [`pack`](#knowledge-packs), [`registry`](#pack-registries) | Knowledge packs and git pack registries | +| [`serve`](#libscope-serve) | MCP server, REST API or web dashboard | +| [`config`](#libscope-config), [`workspace`](#workspaces) | Settings and separate knowledge bases | +| [`webhooks`](#libscope-webhooks) | Signed POSTs on document events | +| [`admin`](#libscope-admin) | Reindex, find duplicates, backup/restore, statistics, prune | +| [`doctor`](#libscope-doctor) | Check the setup; `--fix` creates the database and vector index | -Index a single file or URL. +## Global options -```bash -libscope add ./path/to/doc.md --topic deployment --title "Deploy Guide" -libscope add https://reactjs.org/docs/hooks-intro.html --library react -``` +These go before or after the command name. -| Option | Description | -| ------------------ | --------------------------- | -| `--library ` | Assign to a library | -| `--topic ` | Assign to a topic | -| `--title ` | Override the document title | -| `--version <ver>` | Library version tag | +<!-- generated:start cli:global --> -### `libscope import` +Usage: `libscope [options] <command>` -Recursively import files from a directory. +| Option | Description | +| --------------------- | ------------------------------------------------------------------------------- | +| `-V, --version` | output the version number | +| `--json` | Print results as JSON | +| `-v, --verbose` | Debug logging, and stack traces on errors | +| `--log-level <level>` | Log level (default silent). One of: `debug`, `info`, `warn`, `error`, `silent`. | +| `--workspace <name>` | Use this workspace instead of the active one | -```bash -libscope import ./wiki/ --topic internal --extensions .md,.mdx,.txt -``` +`-h, --help` shows the help of the program or of any command. + +<!-- generated:end cli:global --> + +Every result shows document IDs (and chunk IDs for search hits), so you can pass them to the next command. + +**Errors.** A failed command prints `✗ <message>` and a next step on stderr, and exits with code 1. For example, an unknown document ID suggests `libscope docs list`, and a configuration problem suggests `libscope doctor`. Add `--verbose` to see the stack trace. -| Option | Description | -| --------------------- | ------------------------------------------------ | -| `--library <name>` | Assign all docs to a library | -| `--topic <name>` | Assign all docs to a topic | -| `--extensions <exts>` | Comma-separated file extensions (default: `.md`) | +**Progress.** Long operations (directory imports, crawls, reindexing, pack installs) show a progress line on stderr when it is a terminal. The first command that needs the local embedding model prints one line while the model downloads (about 80 MB, first run only). -### `libscope import-batch` +### Document filters -Parallel batch import with progress tracking. +Commands that select documents (`search`, `ask`, `docs list`, `searches save`, `bulk`) share these flags: + +| Flag | Description | +| ---------------------- | -------------------------------------------------------------- | +| `--topic <topic>` | Topic ID or name | +| `--library <name>` | Library name | +| `--lib-version <version>` | Library version (not on `bulk`) | +| `--source-type <type>` | `library`, `topic`, `manual` or `model-generated` | +| `--tags <a,b>` | Only documents with all these tags | +| `-n, --limit <n>` | Maximum results (`ask`: chunks used as context; not on `bulk`) | + +## `libscope add` + +Each source can be a file, a directory, a URL or a GitHub/GitLab repository URL. LibScope detects which one it is. A local file's absolute path is stored as its URL, so adding a changed file again replaces the old version. ```bash -libscope import-batch ./docs/ --concurrency 10 --filter "**/*.md" --library my-lib +libscope add ./guide.md --topic deployment --tags ops,runbook +libscope add ./docs --include "**/*.md" --exclude "drafts/**" --library my-lib --lib-version 2.1 +libscope add ./docs --watch # keep re-indexing changed files (Ctrl+C to stop) +libscope add https://react.dev/learn --library react +libscope add https://docs.example.com --spider --max-pages 50 --path-prefix /guide +libscope add https://github.com/org/repo --branch main --path docs +libscope add ./docs --dry-run # list what would be added ``` -| Option | Description | -| ------------------- | ---------------------------- | -| `--concurrency <n>` | Number of parallel workers | -| `--filter <glob>` | Glob pattern to match files | -| `--library <name>` | Assign all docs to a library | - -### `libscope search` +<!-- generated:start cli:add --> + +Usage: `libscope add <sources...>` + +Add files, directories, URLs or GitHub/GitLab repository URLs to the knowledge base. + +| Option | Description | +| ------------------------- | ------------------------------------------------------------- | +| `--topic <topic>` | Topic (ID or name) | +| `--library <name>` | Library name | +| `--lib-version <version>` | Library version | +| `--source-type <type>` | Source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Tags to add (comma-separated) | +| `--title <title>` | Title (one file or URL; default: detected) | +| `--format <ext>` | Read files as this format, e.g. .pdf | +| `--dedup <mode>` | Duplicate handling: skip, warn or force | +| `--expires <time>` | ISO 8601 time after which `admin prune` deletes the documents | +| `--include <globs>` | Directory: only files matching these globs (comma-separated) | +| `--exclude <globs>` | Directory: skip files matching these globs (comma-separated) | +| `--watch` | Directory: keep running and re-index files when they change | +| `--spider` | URL: also crawl linked pages | +| `--max-pages <n>` | Crawl: page limit (default 25, max 200) | +| `--max-depth <n>` | Crawl: link depth (default 2, max 5) | +| `--no-same-domain` | Crawl: also follow links to other domains | +| `--path-prefix <path>` | Crawl: only follow links under this path | +| `--exclude-urls <globs>` | Crawl: skip URLs matching these globs (comma-separated) | +| `--branch <name>` | Repository: branch (default: from the URL, else main) | +| `--path <dirs>` | Repository: only these subdirectories (comma-separated) | +| `--extensions <exts>` | Repository: file extensions (default .md,.mdx,.txt,.rst) | +| `--token <token>` | Repository: access token for a private repository | +| `--dry-run` | List what would be added without adding it | + +<!-- generated:end cli:add --> + +With `--json`, `add` prints one result per source: `{ kind, documents: [{ documentId, title, chunkCount, source }], errors, skipped, planned?, crawl? }`. + +## `libscope search` ```bash -libscope search "authentication best practices" --library my-lib --limit 10 -libscope search "deploy process" --context 1 # include neighboring chunks +libscope search "authentication best practices" --library my-lib -n 10 +libscope search "deploy process" --context 1 # show neighbouring chunks +libscope search --related <documentId|chunkId> # content similar to a document or chunk +libscope search # interactive search (on a terminal) ``` -| Option | Description | -| -------------------------- | ----------------------------------------------------------------------------- | -| `--library <name>` | Filter by library | -| `--topic <name>` | Filter by topic | -| `--source <type>` | Filter by source type (e.g., `library`, `topic`, `manual`, `model-generated`) | -| `--limit <n>` | Max results (default: 10) | -| `--min-rating <n>` | Minimum average rating | -| `--max-chunks-per-doc <n>` | Max chunks per document in results (default: no limit) | -| `--context <n>` | Include N neighboring chunks before/after each result (0-2, default: 0) | -| `--save <name>` | Save this search with the given name for later re-use | +<!-- generated:start cli:search --> -### `libscope searches` +Usage: `libscope search [query]` -Manage saved searches. +Search by meaning and keywords. With no query on a terminal, start interactive search. -```bash -libscope searches list # list all saved searches -libscope searches run "My Search" # re-run a saved search by name or ID -libscope searches delete "My Search" # delete a saved search -libscope search "auth best practices" --save "Auth Docs" # save a search while running it -``` +| Option | Description | +| ------------------------- | --------------------------------------------------------------- | +| `--related <id>` | Find content similar to this document or chunk ID instead | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--lib-version <version>` | Only this library version | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `-n, --limit <n>` | Maximum results | +| `--offset <n>` | Results to skip (paging) | +| `--min-rating <n>` | Only documents rated at least this (1-5) | +| `--max-per-doc <n>` | At most this many chunks per document | +| `--context <n>` | Neighbouring chunks to show around each result (0-2) | -| Subcommand | Description | -| ---------------------------- | -------------------------------- | -| `searches list` | List all saved searches | -| `searches run <nameOrId>` | Run a saved search by name or ID | -| `searches delete <nameOrId>` | Delete a saved search | +<!-- generated:end cli:search --> -### `libscope ask` +With no query and no `--related`, on a terminal, `search` starts interactive mode: type a query, see results, repeat. Type `quit`, `exit` or press Ctrl+D to leave. The filters given on the command line apply to every query. + +## `libscope ask` ```bash -libscope ask "How do I configure OAuth2?" --library my-lib --top-k 8 +libscope ask "How do I configure OAuth2?" --library my-lib -n 8 +libscope ask "What changed in v2?" --model gpt-4o-mini ``` -| Option | Description | -| ------------------ | ---------------------------------------- | -| `--library <name>` | Filter source documents by library | -| `--topic <name>` | Filter source documents by topic | -| `--top-k <n>` | Number of chunks to retrieve for context | +<!-- generated:start cli:ask --> -### `libscope serve` +Usage: `libscope ask <question>` -Start the MCP server, REST API, or web dashboard. +Answer a question from the knowledge base with the configured LLM. -```bash -libscope serve # MCP server (stdio) -libscope serve --api # REST API (port 3378) -libscope serve --dashboard # Web dashboard UI (port 3377) -libscope serve --dashboard --port 8080 -``` +| Option | Description | +| ------------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--lib-version <version>` | Only this library version | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `-n, --limit <n>` | Chunks to use as context (default 5) | +| `--min-rating <n>` | Only documents rated at least this (1-5) | +| `--model <model>` | LLM model for this question (default: llm.model) | + +<!-- generated:end cli:ask --> + +`ask` needs an LLM. With `llm.provider` set to `auto` (the default), LibScope uses OpenAI when an OpenAI API key is set, else Anthropic when an Anthropic key is set, else Ollama when `llm.url` is set or the embedding provider is Ollama. Without an LLM, `ask` fails with a hint; `libscope doctor` shows what is resolved. With `llm.provider passthrough`, `ask` prints the retrieved context instead of an answer. + +## `libscope docs` + +<!-- generated:start cli:docs --> + +### `libscope docs list` + +List documents, newest first. + +| Option | Description | +| ------------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--lib-version <version>` | Only this library version | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `-n, --limit <n>` | Maximum results | +| `--offset <n>` | Documents to skip (paging) | + +### `libscope docs show <documentId>` + +Show a document with its tags, links and ratings. + +| Option | Description | +| ------------------ | -------------------------------------------- | +| `--offset <n>` | Start the content at this character | +| `--max-length <n>` | Show at most this many characters of content | + +### `libscope docs history <documentId>` + +List saved versions of a document. + +### `libscope docs update <documentId>` + +Change a document's title, content, metadata or tags. + +| Option | Description | +| ------------------------- | --------------------------------------------- | +| `--title <title>` | New title | +| `--content <text>` | New content (re-chunked and re-embedded) | +| `--content-file <path>` | Read the new content from a file | +| `--library <name>` | New library name | +| `--lib-version <version>` | New library version | +| `--url <url>` | New URL | +| `--topic <topic>` | New topic (ID or name) | +| `--tags <tags>` | Replace the tags with these (comma-separated) | + +### `libscope docs delete <documentId>` + +Delete a document with its chunks, links and ratings. + +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope docs rollback <documentId> <version>` + +Restore a saved version (the current state is saved first). + +### `libscope docs rate <documentId> <rating>` + +Rate a document from 1 (poor) to 5 (excellent). + +| Option | Description | +| --------------------- | ------------------------------ | +| `--feedback <text>` | What is good or wrong | +| `--correction <text>` | Suggested replacement text | +| `--chunk <chunkId>` | Rate one chunk of the document | + +### `libscope docs link <documentId> <targetDocumentId>` + +Link one document to another. + +| Option | Description | +| ---------------- | ----------------------------------------------------------------------------- | +| `--type <type>` | Link type: see_also, prerequisite, supersedes, related, references. Required. | +| `--label <text>` | Short description of the relationship | + +### `libscope docs unlink <linkId>` + +Delete a link (link IDs are shown by `docs links`). + +### `libscope docs links [documentId]` + +List the links of a document, or every link. + +| Option | Description | +| --------------- | ---------------------------------------------------------------------------- | +| `--type <type>` | Only this link type: see_also, prerequisite, supersedes, related, references | + +### `libscope docs prereqs <documentId>` + +List the documents to read first (following prerequisite links). + +### `libscope docs tag <documentId> <tags...>` + +Add tags to a document (space- or comma-separated). -| Option | Description | -| ------------- | ------------------------------------------------------- | -| `--api` | Start REST API instead of MCP server | -| `--dashboard` | Start the web dashboard UI | -| `--port <n>` | Server port (default: 3378 for API, 3377 for dashboard) | -| `--host <h>` | Server host (default: localhost) | +### `libscope docs untag <documentId> <tags...>` -## Document Management +Remove tags from a document. -| Command | Description | -| --------------------------------------- | ------------------------------ | -| `libscope docs list` | List indexed documents | -| `libscope docs show <id>` | Show a specific document | -| `libscope docs update <id>` | Update a document | -| `libscope docs delete <id>` | Delete a document | -| `libscope docs history <id>` | View version history | -| `libscope docs rollback <id> <version>` | Rollback to a previous version | +### `libscope docs suggest-tags <documentId>` -## Document Updates +Suggest tags from a document's content. -### `libscope docs update` +| Option | Description | +| ----------------- | ------------------- | +| `-n, --limit <n>` | Maximum suggestions | -Update an existing document's title, content, or metadata. Changing content triggers re-chunking and re-indexing of embeddings. +<!-- generated:end cli:docs --> + +`docs update --tags` replaces all tags. Changing the content re-chunks and re-embeds the document. ```bash -libscope docs update <documentId> --title "New Title" -libscope docs update <documentId> --content "Updated content here" -libscope docs update <documentId> --library vue --version 3.0.0 +libscope docs update <documentId> --title "New Title" --tags guide,v2 +libscope docs update <documentId> --content-file ./updated.md ``` -| Option | Description | -| --------------------- | ---------------------------------- | -| `--title <title>` | New document title | -| `--content <content>` | New content (triggers re-chunking) | -| `--library <name>` | New library name | -| `--version <ver>` | New library version | -| `--url <url>` | New source URL | -| `--topic <topicId>` | New topic ID | +## Topics and tags -## Bulk Operations +<!-- generated:start cli:topics,tags --> -Perform operations on multiple documents at once using filter criteria. +### `libscope topics list` -### `libscope bulk delete` +List topics with their document counts. + +| Option | Description | +| ------------------ | ------------------------------------------------ | +| `--parent <topic>` | Only direct subtopics of this topic (ID or name) | + +### `libscope topics create <name>` + +Create a topic. + +| Option | Description | +| ---------------------- | ------------------------- | +| `--description <text>` | What the topic covers | +| `--parent <topic>` | Parent topic (ID or name) | + +### `libscope topics delete <topic>` + +Delete a topic (its documents are kept without a topic). + +| Option | Description | +| -------------------- | --------------------------------- | +| `--delete-documents` | Also delete the topic's documents | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope tags list` + +List all tags with their document counts. + +<!-- generated:end cli:topics,tags --> + +`<topic>` is a topic ID or name everywhere. + +## `libscope searches` + +<!-- generated:start cli:searches --> + +### `libscope searches save <name> <query>` + +Save a query and its filters. + +| Option | Description | +| ------------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--lib-version <version>` | Only this library version | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `-n, --limit <n>` | Maximum results | +| `--min-rating <n>` | Only documents rated at least this (1-5) | + +### `libscope searches list` + +List saved searches. + +| Option | Description | +| ----------------- | ------------------------ | +| `-n, --limit <n>` | Maximum results | +| `--offset <n>` | Results to skip (paging) | + +### `libscope searches run <search>` + +Run a saved search (name or ID). + +### `libscope searches delete <search>` + +Delete a saved search (name or ID). -Delete all documents matching the specified filters. +<!-- generated:end cli:searches --> + +## `libscope bulk` + +`bulk delete`, `bulk retag` and `bulk move` change every document that matches the filters. At least one filter is required, and one call changes at most 1000 documents. Each command first shows how many documents match and asks for confirmation. ```bash libscope bulk delete --library react --dry-run -libscope bulk delete --topic topic-1 --source-type manual --yes +libscope bulk retag --topic guides --add important,v2 --remove draft -y +libscope bulk move --library react --since 2024-01-01 --to frontend -y ``` -### `libscope bulk retag` +<!-- generated:start cli:bulk --> -Add or remove tags from all matching documents. +### `libscope bulk delete` -```bash -libscope bulk retag --library react --add-tags important,v2 --dry-run -libscope bulk retag --topic topic-1 --remove-tags deprecated --yes -``` +Delete matching documents. + +| Option | Description | +| ---------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `--since <date>` | Only documents created on or after (ISO 8601) | +| `--before <date>` | Only documents created on or before (ISO 8601) | +| `--dry-run` | List the matching documents without changing them | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope bulk retag` + +Add and/or remove tags on matching documents. + +| Option | Description | +| ---------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `--since <date>` | Only documents created on or after (ISO 8601) | +| `--before <date>` | Only documents created on or before (ISO 8601) | +| `--dry-run` | List the matching documents without changing them | +| `-y, --yes` | Do not ask for confirmation | +| `--add <tags>` | Tags to add (comma-separated) | +| `--remove <tags>` | Tags to remove (comma-separated) | ### `libscope bulk move` -Move all matching documents to a different topic. +Move matching documents to another topic. -```bash -libscope bulk move --library react --to new-topic-id --dry-run -libscope bulk move --topic old-topic --to new-topic --yes -``` +| Option | Description | +| ---------------------- | --------------------------------------------------------------- | +| `--topic <topic>` | Only this topic (ID or name) | +| `--library <name>` | Only this library | +| `--source-type <type>` | Only this source type (library, topic, manual, model-generated) | +| `--tags <tags>` | Only documents with all these tags (comma-separated) | +| `--since <date>` | Only documents created on or after (ISO 8601) | +| `--before <date>` | Only documents created on or before (ISO 8601) | +| `--dry-run` | List the matching documents without changing them | +| `-y, --yes` | Do not ask for confirmation | +| `--to <topic>` | Destination topic (ID or name). Required. | -| Option | Description | -| ---------------------- | ------------------------------------------------- | -| `--topic <topicId>` | Filter by topic ID | -| `--library <name>` | Filter by library name | -| `--source-type <type>` | Filter by source type | -| `--tags <tags>` | Filter by tags (comma-separated) | -| `--to <targetTopicId>` | Target topic (move only) | -| `--add-tags <tags>` | Tags to add (retag only, comma-separated) | -| `--remove-tags <tags>` | Tags to remove (retag only, comma-separated) | -| `--dry-run` | Preview affected documents without making changes | -| `-y, --yes` | Skip confirmation prompt | - -## Document Links (Cross-references) - -| Command | Description | -| ------------------------------------- | ------------------------------- | -| `libscope link <sourceId> <targetId>` | Create a cross-reference link | -| `libscope links <documentId>` | Show all links for a document | -| `libscope unlink <linkId>` | Remove a link | -| `libscope prereqs <documentId>` | Show prerequisite reading chain | - -### `libscope link` +<!-- generated:end cli:bulk --> + +## Connectors + +A connection is a saved connector configuration in `~/.libscope/connectors/<name>.json` (mode `0600`, credentials included). + +Connector types: `notion`, `slack`, `confluence`, `obsidian`, `onenote`, `docs` (a documentation site). `[source]` is the vault path for `obsidian` and the URL for `confluence` and `docs`. ```bash -libscope link <sourceId> <targetId> --type see_also --label "Background context" +libscope connect notion --token secret_xxx +libscope connect slack --token xoxb-xxx --channels general,eng --exclude random +libscope connect confluence https://acme.atlassian.net --email me@acme.com --token xxx --spaces ENG +libscope connect obsidian ~/notes --topic-mapping frontmatter +libscope connect onenote --client-id <azure-app-id> # device-code sign-in +libscope connect docs https://docs.example.com --library example --max-pages 200 +libscope connect notion --name work --schedule "0 */6 * * *" # second Notion connection, synced every 6 hours ``` -| Option | Description | -| ---------------- | ----------------------------------------------------------------------------------- | -| `--type <type>` | Link type: `see_also`, `prerequisite`, `supersedes`, `related` (default: `related`) | -| `--label <text>` | Optional description of the relationship | +Running `connect` again for a saved connection changes only the settings you give and keeps the rest. For example, `libscope connect notion --schedule off --no-sync` removes the schedule. -## Topics & Tags +<!-- generated:start cli:connect,sync,disconnect,connections --> -| Command | Description | -| ------------------------------------ | -------------------------------------------- | -| `libscope topics list` | List all topics | -| `libscope topics create <name>` | Create a topic (`--parent`, `--description`) | -| `libscope tag add <docId> <tags...>` | Add tags to a document | -| `libscope tag remove <docId> <tag>` | Remove a tag | -| `libscope tag list` | List all tags with document counts | +### `libscope connect <type> [source]` -## Workspaces +Save a connection and sync it. Types: notion, slack, confluence, obsidian, onenote, docs. [source] is the vault path (obsidian) or the URL (confluence, docs). Running it again for a saved connection changes only the given settings. + +| Option | Description | +| ------------------------- | ----------------------------------------------------------------------------------- | +| `--name <name>` | Connection name (default: the type) | +| `--schedule <cron>` | Sync on this cron schedule while `libscope serve api` runs ("off" removes it) | +| `--no-sync` | Save the connection without syncing now | +| `--token <token>` | notion, slack, confluence: API token; onenote: access token | +| `--exclude <list>` | Comma-separated: slack channels, notion page IDs, confluence spaces, obsidian globs | +| `--channels <list>` | slack: channel names or IDs (default all) | +| `--thread-mode <mode>` | slack: aggregate (one document per thread) or separate | +| `--email <email>` | confluence: user email (Cloud) | +| `--server` | confluence: Server/Data Center instead of Cloud | +| `--spaces <keys>` | confluence: space keys (default all) | +| `--topic-mapping <mode>` | obsidian: topics from folder (default) or frontmatter | +| `--notebook <name>` | onenote: one notebook (default all) | +| `--client-id <id>` | onenote: Azure app client ID for device sign-in | +| `--tenant-id <id>` | onenote: Azure tenant ID (default common) | +| `--site-type <type>` | docs: auto, sphinx, vitepress, doxygen or generic | +| `--library <name>` | docs: library name for the pages | +| `--lib-version <version>` | docs: library version for the pages | +| `--max-pages <n>` | docs: page limit (default 500) | +| `--max-depth <n>` | docs: link depth (default 10) | +| `--path-prefix <path>` | docs: only pages under this path | + +### `libscope sync [name]` -| Command | Description | -| ---------------------------------- | ----------------------- | -| `libscope workspace create <name>` | Create a new workspace | -| `libscope workspace list` | List all workspaces | -| `libscope workspace use <name>` | Switch active workspace | -| `libscope workspace delete <name>` | Delete a workspace | +Sync a saved connection with its saved settings, or all with --all. -## Knowledge Packs +| Option | Description | +| ------- | --------------------------- | +| `--all` | Sync every saved connection | -| Command | Description | -| ------------------------------------ | ------------------------------------------------- | -| `libscope pack install <nameOrPath>` | Install a pack (from registry, file, or by name) | -| `libscope pack remove <name>` | Remove a pack and its documents | -| `libscope pack list` | List installed packs (`--available` for registry) | -| `libscope pack create` | Export documents as a pack file | +### `libscope disconnect <name>` -### `libscope pack install` (registry support) +Delete a connection's documents and its saved settings (including credentials). -When no local file path is given, `pack install` searches configured registries: +| Option | Description | +| ------------------ | ------------------------------------------------------ | +| `--type <type>` | Connector type, when no saved connection has this name | +| `--keep-documents` | Only delete the saved settings | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope connections` + +List saved connections with their schedule and last sync. + +<!-- generated:end cli:connect,sync,disconnect,connections --> + +## Knowledge packs ```bash -libscope pack install react-docs # latest from any registry -libscope pack install react-docs@1.2.0 # specific version -libscope pack install react-docs --version 1.2.0 -libscope pack install react-docs --registry official +libscope pack install react-docs # latest version +libscope pack install react-docs@1.2.0 +libscope pack install react-docs --registry official # when several registries have react-docs +libscope pack install ./react-docs.json.gz +libscope pack create --name team-docs --topic engineering +libscope pack create --name react-docs --from ./react/docs --exclude "*.min.js" ``` -| Option | Description | -| -------------------- | -------------------------------------------- | -| `--version <semver>` | Install a specific version (default: latest) | -| `--registry <name>` | Install from a specific registry | +Pack names are looked up in the local copies of the registries; run `libscope registry sync` to update them. When more than one registry has the pack, the command fails and names the registries: choose one with `--registry`. + +<!-- generated:start cli:pack --> + +### `libscope pack install <pack>` -## Pack Registries +Install a pack from the configured registries (name or name@version) or a local .json/.json.gz file. -| Command | Description | -| ---------------------------------------------------------- | -------------------------------------- | -| `libscope registry add <url> [-n <alias>]` | Register a git repo as a pack registry | -| `libscope registry remove <name> [-y]` | Unregister a registry | -| `libscope registry list` | List configured registries | -| `libscope registry sync [<name>]` | Sync one or all registries | -| `libscope registry search <query> [-r <name>]` | Search registry pack indexes | -| `libscope registry create <path>` | Initialize a new registry repo | -| `libscope registry publish <file> -r <name>` | Publish a pack file to a registry | -| `libscope registry unpublish <pack> -r <name> --version <v>` | Remove a pack version from a registry | +| Option | Description | +| ------------------- | ----------------------------------------------------- | +| `--registry <name>` | Look only in this registry | +| `--batch-size <n>` | Documents embedded per batch (default 10) | +| `--resume-from <n>` | Skip the first N documents (resume a partial install) | +| `--concurrency <n>` | Batches embedded in parallel (default 4) | -### `libscope registry add` +### `libscope pack remove <name>` + +Remove an installed pack and its documents. + +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope pack list` + +List installed packs, or the packs in the configured registries (--available). + +| Option | Description | +| ------------------- | ------------------------------------------- | +| `--available` | List the packs in the configured registries | +| `--registry <name>` | With --available: only this registry | + +### `libscope pack create` + +Create a pack from indexed documents, or from files, folders and URLs (--from). + +| Option | Description | +| -------------------------- | ---------------------------------------------------------------------------- | +| `--name <name>` | Pack name. Required. | +| `--from <sources...>` | Files, folders or URLs to build the pack from | +| `--topic <topic>` | Only this topic's documents (without --from) | +| `--pack-version <version>` | Pack version (default 1.0.0) | +| `--description <text>` | Pack description | +| `--author <name>` | Pack author | +| `--license <license>` | Pack license | +| `--output <path>` | Output file (default <name>.json, or <name>.json.gz with --from) | +| `--extensions <exts>` | With --from: file extensions to include (comma-separated) | +| `--exclude <globs...>` | With --from: globs to skip | +| `--no-recursive` | With --from: do not walk subdirectories | + +<!-- generated:end cli:pack --> + +## Pack registries + +A registry is a git repository of packs. `--registry <name>` always means the name of a configured registry. ```bash libscope registry add https://github.com/org/registry.git -libscope registry add git@github.com:team/packs.git --name team --priority 5 -libscope registry add https://github.com/org/registry.git --sync-interval 86400 --no-sync +libscope registry add git@github.com:team/packs.git --name team +libscope registry add file:///srv/libscope-packs.git --no-sync +libscope registry publish ./my-pack.json --registry my-registry --pack-version 1.0.0 +libscope registry publish ./my-pack.json.gz --registry my-registry # next patch version +libscope registry publish ./my-pack.json --registry community --submit # push to a feature branch +libscope registry unpublish my-pack@1.0.0 --registry my-registry ``` -| Option | Description | -| ---------------------------- | -------------------------------------------------------- | -| `-n, --name <alias>` | Short name for this registry (default: inferred from URL)| -| `--priority <n>` | Conflict resolution priority — lower wins (default: 10) | -| `--sync-interval <seconds>` | Auto-sync interval in seconds, 0 = manual (default: 0) | -| `--no-sync` | Skip initial sync after adding | +Registry URLs can use `https://`, `ssh://`, `git@host:path` or `file:///`. `libscope registry create ./my-registry` creates a git repository with the registry folder structure. See the [Registry Reference](/reference/registry). -### `libscope registry search` +<!-- generated:start cli:registry --> -```bash -libscope registry search "react" -libscope registry search "kubernetes" -r official -``` +### `libscope registry add <url>` -### `libscope registry publish` +Add a git repository as a pack registry and clone it. -```bash -# Direct publish (you have write access) -libscope registry publish ./my-pack.json -r my-registry --version 1.0.0 +| Option | Description | +| ------------------- | ------------------------------------- | +| `-n, --name <name>` | Registry name (default: from the URL) | +| `--no-sync` | Do not clone the registry now | -# Auto-bump patch version -libscope registry publish ./my-pack.json -r my-registry +### `libscope registry remove <name>` -# Submit via feature branch (for PR workflow) -libscope registry publish ./my-pack.json -r community --submit -``` +Remove a registry and delete its local clone. + +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope registry list` + +List the configured registries. + +### `libscope registry sync [name]` + +Fetch the latest packs of one or all registries. + +### `libscope registry search <query>` + +Search the packs in the configured registries. + +| Option | Description | +| ------------------- | ------------------------- | +| `--registry <name>` | Search only this registry | + +### `libscope registry create <path>` + +Create an empty registry git repository. + +### `libscope registry publish <file>` -| Option | Description | -| ------------------------ | -------------------------------------------------------- | -| `-r, --registry <name>` | Target registry (required) | -| `--version <semver>` | Version to publish as (default: auto-bump patch) | -| `-m, --message <msg>` | Git commit message | -| `--submit` | Push to a feature branch instead of main | +Publish a pack file to a registry (commit and push). -### `libscope registry unpublish` +| Option | Description | +| ------------------------- | ---------------------------------------------------------------------- | +| `--registry <name>` | Target registry. Required. | +| `--pack-version <semver>` | Version to publish (default: next patch version) | +| `-m, --message <msg>` | Git commit message | +| `--submit` | Push to a feature branch for a pull request instead of the main branch | + +### `libscope registry unpublish <pack>` + +Remove one version of a pack from a registry (<name>@<version>). + +| Option | Description | +| --------------------- | --------------------------- | +| `--registry <name>` | Target registry. Required. | +| `-m, --message <msg>` | Git commit message | +| `-y, --yes` | Do not ask for confirmation | + +<!-- generated:end cli:registry --> + +## `libscope serve` ```bash -libscope registry unpublish my-pack -r my-registry --version 1.0.0 +libscope serve # MCP server on stdio +libscope serve api # REST API on http://localhost:3378 +libscope serve dashboard --port 8080 # web dashboard (default port 3377) ``` -| Option | Description | -| ------------------------ | --------------------------------- | -| `-r, --registry <name>` | Target registry (required) | -| `--version <semver>` | Version to remove (required) | -| `-m, --message <msg>` | Git commit message | -| `-y, --yes` | Skip confirmation prompt | +<!-- generated:start cli:serve --> + +Usage: `libscope serve [mode]` + +Start the MCP server on stdio (mcp, default), the REST API (api, port 3378) or the web dashboard (dashboard, port 3377). + +| Option | Description | +| --------------- | ------------------------------------- | +| `--port <n>` | Port (api and dashboard) | +| `--host <host>` | Host to listen on (default localhost) | + +<!-- generated:end cli:serve --> + +Connection schedules (`connect --schedule`) run while `serve api` runs. + +## `libscope config` + +<!-- generated:start cli:config --> + +### `libscope config show` -### `libscope registry create` +Show the effective configuration (API keys are masked). + +### `libscope config get <key>` + +Print the effective value of a key (API keys are masked). + +### `libscope config set <key> <value>` + +Set a key in the user config file. Keys: embedding.provider, embedding.model, embedding.url, embedding.dimensions, llm.provider, llm.model, llm.url, openai.apiKey, anthropic.apiKey, database.path, indexing.maxDocumentSize, indexing.allowPrivateUrls, indexing.allowSelfSignedCerts, logging.level, mcp.toolsets. API keys are written to ~/.libscope/secrets.json (mode 0600), never to config.json. + +### `libscope config unset <key>` + +Remove a key from the user config file (or secrets file for API keys). + +### `libscope config path` + +Print the path of the user config file. + +<!-- generated:end cli:config --> + +`openai.apiKey` and `anthropic.apiKey` are written to `~/.libscope/secrets.json` (mode `0600`), never to `config.json`; the `LIBSCOPE_OPENAI_API_KEY` / `OPENAI_API_KEY` and `LIBSCOPE_ANTHROPIC_API_KEY` / `ANTHROPIC_API_KEY` environment variables take precedence. See the [configuration reference](/reference/configuration) for every key. + +## Workspaces + +A workspace is a separate knowledge base with its own database (`~/.libscope/workspaces/<name>/libscope.db`). The global `--workspace <name>` option and the `LIBSCOPE_WORKSPACE` environment variable select one for a single command. + +<!-- generated:start cli:workspace --> + +### `libscope workspace create <name>` + +Create a workspace. + +### `libscope workspace list` + +List workspaces. + +### `libscope workspace use <name>` + +Make a workspace the active one. + +### `libscope workspace delete <name>` + +Delete a workspace and its database. + +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | + +<!-- generated:end cli:workspace --> + +## `libscope webhooks` + +<!-- generated:start cli:webhooks --> + +### `libscope webhooks list` + +List webhooks (secrets are never shown). + +### `libscope webhooks create <url>` + +Register a webhook. + +| Option | Description | +| ------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `--events <events>` | Comma-separated: document.created, document.updated, document.deleted, document.rated, search.executed. Required. | +| `--secret <secret>` | HMAC-SHA256 signing secret (needs LIBSCOPE_SECRET_KEY) | + +### `libscope webhooks delete <webhookId>` + +Delete a webhook. + +### `libscope webhooks test <webhookId>` + +Send a test event and show the HTTP status. + +<!-- generated:end cli:webhooks --> + +Events: `document.created`, `document.updated`, `document.deleted`, `document.rated`, `search.executed`. A secret signs each POST with HMAC-SHA256; storing it needs `LIBSCOPE_SECRET_KEY`. + +## `libscope admin` + +<!-- generated:start cli:admin --> + +### `libscope admin reindex` + +Re-embed chunks with the configured embedding model. + +| Option | Description | +| ------------------------ | --------------------------------------------------------------------- | +| `--rebuild` | Recreate the vector index for the configured model, then re-embed all | +| `--doc <documentIds...>` | Only these documents | +| `--since <date>` | Only documents created on or after (ISO 8601) | +| `--before <date>` | Only documents created on or before (ISO 8601) | +| `--batch-size <n>` | Chunks per embedding call (default 50) | + +### `libscope admin dedupe` + +Find duplicate and near-duplicate documents. + +| Option | Description | +| ----------------------- | --------------------------------------- | +| `--threshold <n>` | Similarity threshold 0-1 (default 0.95) | +| `--strategy <strategy>` | exact, semantic or both (default) | + +### `libscope admin prune` + +Delete documents whose expiry time has passed. + +### `libscope admin backup <file>` + +Write the whole knowledge base to a JSON file. + +### `libscope admin restore <file>` + +Import a file written by `admin backup`. + +| Option | Description | +| ----------- | --------------------------- | +| `-y, --yes` | Do not ask for confirmation | + +### `libscope admin stats` + +Counts, index, most returned and stale documents, and search analytics. + +| Option | Description | +| ------------ | ------------------------------------------------------- | +| `--days <n>` | Look-back days for stale documents and search analytics | + +<!-- generated:end cli:admin --> + +`admin reindex --rebuild` recreates the vector index for a new embedding model or vector size, then re-embeds every chunk. The other `admin reindex` filters do not apply with `--rebuild`. + +## `libscope doctor` ```bash -libscope registry create ./my-registry +libscope doctor # report only +libscope doctor --fix # also create the database and vector index ``` -Creates a git repo with the canonical registry folder structure. See the [Registry Reference](/reference/registry) for full schema details. +<!-- generated:start cli:doctor --> -## Connectors +Usage: `libscope doctor` + +Check the setup: config, workspace, database, embedding model, index and LLM. + +| Option | Description | +| ------- | -------------------------------------------------- | +| `--fix` | Create what is missing (database and vector index) | + +<!-- generated:end cli:doctor --> -| Command | Description | -| ---------------------------------- | -------------------------------- | -| `libscope connect obsidian <path>` | Sync an Obsidian vault | -| `libscope connect onenote` | Sync OneNote notebooks | -| `libscope connect notion` | Sync Notion pages and databases | -| `libscope connect confluence` | Sync Confluence spaces | -| `libscope connect slack` | Sync Slack channels | -| `libscope add-repo <url>` | Index a GitHub/GitLab repository | -| `libscope disconnect <connector>` | Remove connector data | - -## Developer Tools - -| Command | Description | -| ------------------------------------- | --------------------------------------- | -| `libscope watch <directory>` | Watch for file changes and auto-reindex | -| `libscope reindex` | Re-embed chunks with current provider | -| `libscope dedupe` | Scan for duplicate documents | -| `libscope export <outputPath>` | Export knowledge base to JSON | -| `libscope import-backup <backupPath>` | Import from a backup file | - -## Analytics - -| Command | Description | -| ------------------------ | --------------------------------- | -| `libscope stats` | Overview dashboard | -| `libscope stats popular` | Most-returned documents in search | -| `libscope stats stale` | Documents with no search hits | -| `libscope stats queries` | Top search queries | - -## Configuration - -| Command | Description | -| ----------------------------------- | -------------------------- | -| `libscope config set <key> <value>` | Set a configuration value | -| `libscope config show` | Show current configuration | - -Supported config keys for `set`: `embedding.provider`, `embedding.ollamaUrl`, `embedding.ollamaModel`, `embedding.openaiModel`, `llm.provider`, `llm.model`, `database.path`, `logging.level`, `indexing.allowPrivateUrls`, `indexing.allowSelfSignedCerts`. - -## Global Options - -These work with any command: - -| Flag | Description | -| --------------------- | -------------------------------------------------------------- | -| `--verbose` | Enable debug logging | -| `--log-level <level>` | Set log level (`debug` / `info` / `warn` / `error` / `silent`) | -| `--workspace <name>` | Use a specific workspace | +`doctor` shows the config and secrets files, the active workspace and database path, the configured embedding model and the model the vector index was built with, and which LLM `ask` will use. It then lists checks with a fix for each warning or error, for example `libscope admin reindex --rebuild` when the index was built with a different embedding model. It exits with code 1 when a check fails. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 30e06aa7..70b22a6b 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1,89 +1,102 @@ # Configuration Reference -Complete reference for all configuration options. +Complete reference for all configuration options. One schema (`src/config-schema.ts`) defines every key, its default, its validation, and its environment variable. Keys renamed in 2.0 are listed in the [migration guide](../migration-v2.md#configuration). ## Config File Locations -| Location | Scope | Precedence | -| ------------------------------- | ------- | ---------- | -| Environment variables | Session | Highest | -| `.libscope.json` (project root) | Project | Medium | -| `~/.libscope/config.json` | User | Low | -| Built-in defaults | Global | Lowest | +| Location | Holds | Precedence | +| ------------------------------- | --------------- | ---------- | +| Environment variables | Any key | Highest | +| `~/.libscope/secrets.json` | API keys only | | +| `.libscope.json` (project root) | Non-secret keys | | +| `~/.libscope/config.json` | Non-secret keys | | +| Built-in defaults | — | Lowest | + +API keys are read only from environment variables and `~/.libscope/secrets.json` (mode `0600`). An API key in `config.json` or `.libscope.json` is ignored, and LibScope logs a warning. + +A value in a config file that fails validation is ignored with a warning, and the next layer applies. An environment variable with an invalid value is also ignored with a warning. ## All Config Keys -### Embedding - -| Key | Type | Default | Description | -| ----------------------- | ------ | -------------------------- | ------------------------------ | -| `embedding.provider` | string | `"local"` | `local`, `ollama`, or `openai` | -| `embedding.ollamaUrl` | string | `"http://localhost:11434"` | Ollama server URL | -| `embedding.ollamaModel` | string | `"nomic-embed-text"` | Ollama embedding model | -| `embedding.openaiModel` | string | `"text-embedding-3-small"` | OpenAI embedding model | - -### LLM (for RAG) - -| Key | Type | Default | Description | -| ----------------- | ------ | ------- | ---------------------------------------------- | -| `llm.provider` | string | — | `openai`, `ollama`, `anthropic`, or `passthrough` | -| `llm.model` | string | — | Model name override | -| `llm.ollamaUrl` | string | — | Ollama server URL (overrides embedding URL) | -| `llm.anthropicApiKey` | string | — | Anthropic API key (or use `LIBSCOPE_ANTHROPIC_API_KEY` env var) | - -### Database - -| Key | Type | Default | Description | -| --------------- | ------ | --------------------------- | --------------------------- | -| `database.path` | string | `"~/.libscope/libscope.db"` | Path to the SQLite database | - -### Logging - -| Key | Type | Default | Description | -| --------------- | ------ | -------- | --------------------------------------------- | -| `logging.level` | string | `"info"` | `debug`, `info`, `warn`, `error`, or `silent` | - -### Indexing - -| Key | Type | Default | Description | -| ------------------------------- | ------- | ------- | ------------------------------------------------- | -| `indexing.allowPrivateUrls` | boolean | `false` | Allow fetching from private/internal IP addresses | -| `indexing.allowSelfSignedCerts` | boolean | `false` | Accept self-signed or untrusted TLS certificates | - -## Environment Variables - -| Variable | Maps to | Default | -| ---------------------------------- | ------------------------------- | ------------------------ | -| `LIBSCOPE_EMBEDDING_PROVIDER` | `embedding.provider` | `local` | -| `LIBSCOPE_OPENAI_API_KEY` | OpenAI API key | — | -| `LIBSCOPE_OLLAMA_URL` | `embedding.ollamaUrl` | `http://localhost:11434` | -| `LIBSCOPE_LLM_PROVIDER` | `llm.provider` | — | -| `LIBSCOPE_LLM_MODEL` | `llm.model` | — | -| `LIBSCOPE_ANTHROPIC_API_KEY` | `llm.anthropicApiKey` | — | -| `LIBSCOPE_ALLOW_PRIVATE_URLS` | `indexing.allowPrivateUrls` | `false` | -| `LIBSCOPE_ALLOW_SELF_SIGNED_CERTS` | `indexing.allowSelfSignedCerts` | `false` | -| `ONENOTE_CLIENT_ID` | OneNote app client ID | — | -| `ONENOTE_TENANT_ID` | OneNote tenant ID | `common` | -| `NOTION_TOKEN` | Notion integration token | — | -| `CONFLUENCE_URL` | Confluence base URL | — | -| `CONFLUENCE_EMAIL` | Confluence user email | — | -| `CONFLUENCE_TOKEN` | Confluence API token | — | +Every key `section.field` has the environment variable `LIBSCOPE_<SECTION>_<FIELD>` (for example `embedding.model` → `LIBSCOPE_EMBEDDING_MODEL`). This table is generated from the schema (`npm run docs:gen`). + +<!-- generated:start config:keys --> + +| Key | Type | Default | Environment variables | Secret | Description | +| ------------------------------- | -------------------------------------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `embedding.provider` | `local` \| `ollama` \| `openai` | `local` | `LIBSCOPE_EMBEDDING_PROVIDER` | | Embedding provider. | +| `embedding.model` | string | ollama: nomic-embed-text; openai: text-embedding-3-small | `LIBSCOPE_EMBEDDING_MODEL` | | Embedding model. Ignored by the local provider (always Xenova/all-MiniLM-L6-v2). | +| `embedding.url` | string | `http://localhost:11434` | `LIBSCOPE_EMBEDDING_URL` | | Server URL of the embedding provider. Used by the ollama provider. | +| `embedding.dimensions` | integer | size of the model | `LIBSCOPE_EMBEDDING_DIMENSIONS` | | Vector size, for models whose size is not built in. | +| `llm.provider` | `auto` \| `openai` \| `anthropic` \| `ollama` \| `passthrough` | `auto` | `LIBSCOPE_LLM_PROVIDER` | | LLM for "ask". "auto": passthrough under MCP, else openai if an OpenAI key is set, else anthropic if an Anthropic key is set, else ollama if llm.url is set or embedding.provider is ollama, else none. | +| `llm.model` | string | openai: gpt-4o-mini; anthropic: claude-3-5-haiku-20241022; ollama: llama3.2 | `LIBSCOPE_LLM_MODEL` | | LLM model. | +| `llm.url` | string | embedding.url, then `http://localhost:11434` | `LIBSCOPE_LLM_URL` | | Server URL of the LLM. Used by the ollama provider. | +| `openai.apiKey` | string | — | `LIBSCOPE_OPENAI_API_KEY`, `OPENAI_API_KEY` | yes | OpenAI API key (embeddings and LLM). | +| `anthropic.apiKey` | string | — | `LIBSCOPE_ANTHROPIC_API_KEY`, `ANTHROPIC_API_KEY` | yes | Anthropic API key (LLM). | +| `database.path` | string | `~/.libscope/workspaces/<workspace>/libscope.db` | `LIBSCOPE_DATABASE_PATH` | | SQLite database file. A leading ~ is expanded. Unset: the workspace database. | +| `indexing.maxDocumentSize` | integer | `104857600` | `LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE` | | Largest document to index, in bytes. | +| `indexing.allowPrivateUrls` | boolean | `false` | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS` | | Allow fetching URLs on private or local networks. | +| `indexing.allowSelfSignedCerts` | boolean | `false` | `LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS` | | Accept self-signed TLS certificates when fetching. | +| `logging.level` | `debug` \| `info` \| `warn` \| `error` \| `silent` | `info` | `LIBSCOPE_LOGGING_LEVEL` | | Log level. | +| `mcp.toolsets` | list of `core` \| `admin` \| `all` | `none` | `LIBSCOPE_MCP_TOOLSETS` | | Optional MCP toolsets (comma list). admin adds sync, install-pack, list-packs and reindex-documents; all enables every optional toolset. The core tools are always on. | + +<!-- generated:end config:keys --> + +Notes: + +- `embedding.dimensions` is needed only for Ollama or OpenAI models whose vector size LibScope does not know. For `text-embedding-3-*`, it also requests shorter vectors. After you change the embedding provider, model or dimensions, run `libscope admin reindex --rebuild`. +- `llm.provider`: see [Automatic LLM selection](#automatic-llm-selection). +- Each API key is read in this order: `LIBSCOPE_<PROVIDER>_API_KEY`, then `<PROVIDER>_API_KEY`, then `~/.libscope/secrets.json`. `libscope config set openai.apiKey <key>` writes the key to `secrets.json`, never to `config.json`. `libscope config show` and `libscope config get` mask keys (for example `sk-…abcd`). +- Booleans accept `true`, `false`, `1` and `0` in environment variables and in `config set`. +- A list key (`mcp.toolsets`) accepts a JSON array in a config file, or a comma-separated list in a config file, an environment variable, and `config set` (for example `libscope config set mcp.toolsets admin`). + +## Automatic LLM selection + +When `llm.provider` is `auto` (the default), LibScope selects the LLM when `ask` runs. It does not make network calls to decide: + +1. Under the MCP server: `passthrough`. The calling assistant writes the answer from the retrieved context. +2. Otherwise, `openai` if an OpenAI key is set. +3. Otherwise, `anthropic` if an Anthropic key is set. +4. Otherwise, `ollama` if `llm.url` is set or `embedding.provider` is `ollama`. +5. Otherwise there is no LLM. `ask` fails with a message that tells you which key or setting to add. + +Set `llm.provider` to a specific provider to skip this selection. + +## Other Environment Variables + +These variables are not config keys: + +| Variable | Description | Default | +| ------------------------- | ----------------------------------------------------------------------------------- | --------- | +| `LIBSCOPE_WORKSPACE` | Active workspace (overrides `libscope workspace use`) | `default` | +| `LIBSCOPE_API_KEY` | REST API key. When set, requests must send `Authorization: Bearer <key>` | — | +| `LIBSCOPE_SECRET_KEY` | Key used to encrypt stored webhook secrets | — | +| `LIBSCOPE_VERBOSE` | `1` turns on debug logs (JSON, on stderr) in the CLI | — | +| `LIBSCOPE_GIT_TIMEOUT_MS` | Timeout of each git call made for pack registries, in milliseconds (at most 300000) | `60000` | + +Connector credentials are not read from environment variables. Give them to `libscope connect <type>` (see [Connectors](/guide/connectors)). + +LibScope does not load `.env` files. Export variables in your shell, or set them in the `env` block of your MCP client config. ## Setting Values ```bash # Embedding libscope config set embedding.provider ollama # local | ollama | openai -libscope config set embedding.ollamaUrl http://localhost:11434 -libscope config set embedding.ollamaModel nomic-embed-text -libscope config set embedding.openaiModel text-embedding-3-small +libscope config set embedding.url http://localhost:11434 +libscope config set embedding.model nomic-embed-text # LLM (for RAG) -libscope config set llm.provider openai # openai | ollama | passthrough +libscope config set llm.provider openai # auto | openai | anthropic | ollama | passthrough libscope config set llm.model gpt-4o-mini +# API keys (written to ~/.libscope/secrets.json) +libscope config set openai.apiKey sk-... +libscope config set anthropic.apiKey sk-ant-... + # Database -libscope config set database.path /custom/path/libscope.db +libscope config set database.path ~/kb/libscope.db # overrides the workspace database +libscope config unset database.path # back to the workspace database # Logging libscope config set logging.level debug # debug | info | warn | error | silent @@ -92,10 +105,23 @@ libscope config set logging.level debug # debug | info | warn | error libscope config set indexing.allowPrivateUrls true libscope config set indexing.allowSelfSignedCerts true -# View current config +# Size limit (bytes) +libscope config set indexing.maxDocumentSize 52428800 + +# MCP server: enable the admin toolset +libscope config set mcp.toolsets admin + +# Read, remove, and locate +libscope config get embedding.provider # effective value (API keys masked) +libscope config unset llm.model # remove from the file that holds the key +libscope config path # print the user config file path + +# View current config (API keys masked) libscope config show ``` +`config set` accepts every key in [All Config Keys](#all-config-keys). It checks the value type (booleans accept `true`/`false`/`1`/`0`, integers must be positive, and enum keys must use a listed value). It writes only the key you set and keeps all other content of the file, including `registries`. API keys go to `~/.libscope/secrets.json`. All other keys go to `~/.libscope/config.json`. Both files are written with mode `0600`. + ## Corporate / Internal Networks If you're indexing docs from internal servers (Confluence, wikis, etc.): @@ -114,16 +140,13 @@ libscope config set indexing.allowSelfSignedCerts true { "embedding": { "provider": "ollama", - "ollamaUrl": "http://localhost:11434", - "ollamaModel": "nomic-embed-text" + "url": "http://localhost:11434", + "model": "nomic-embed-text" }, "llm": { "provider": "openai", "model": "gpt-4o-mini" }, - "database": { - "path": "~/.libscope/libscope.db" - }, "logging": { "level": "info" }, diff --git a/docs/reference/lite-api.md b/docs/reference/lite-api.md index 2f683e54..f1e80e80 100644 --- a/docs/reference/lite-api.md +++ b/docs/reference/lite-api.md @@ -1,590 +1,150 @@ # LibScope Lite API Reference -Complete TypeScript API reference for `libscope/lite`. +TypeScript API of `libscope/lite`. It exports everything from `libscope` (the `LibScope` class, its types and the error classes) and the items on this page. For the `LibScope` methods, see [Programmatic Usage](/guide/programmatic-usage). ## Import ```ts -import { LibScopeLite, TreeSitterChunker } from "libscope/lite"; +import { + createLite, + createCodeChunker, + TreeSitterChunker, + normalizeRawInput, + LibScope, + ConfigError, +} from "libscope/lite"; import type { LiteOptions, - LiteDoc, - RawInput, - LiteSearchOptions, - LiteSearchResult, - LiteContextOptions, - LiteAskOptions, + Chunker, CodeChunk, + RawInput, + NormalizedInput, + EmbeddingProvider, + LlmProvider, + SearchOutput, + AskOutput, } from "libscope/lite"; ``` --- -## `LibScopeLite` - -The main class. Creates and manages its own SQLite database, embedding provider, and search engine. - -### Constructor +## `createLite(options)` ```ts -new LibScopeLite(opts?: LiteOptions) -``` - -**`LiteOptions`** +function createLite(options: LiteOptions): LibScope; -```ts interface LiteOptions { - /** - * Path to the SQLite database file. - * - Use ":memory:" for in-process ephemeral storage (lost on close()) - * - Use a file path for persistent cross-session storage - * - Defaults to ~/.libscope/lite.db - */ - dbPath?: string; - - /** - * Inject a pre-configured better-sqlite3 Database instance. - * When provided, dbPath is ignored. No migrations, no sqlite-vec - * setup, and no extension loading are performed — the caller is - * responsible for schema initialization. - * - * Useful for tests and for callers that already manage their own - * database connection. - */ - db?: Database; - - /** - * Embedding provider used for indexing and similarity search. - * Defaults to LocalEmbeddingProvider (all-MiniLM-L6-v2, ~80 MB download). - */ - provider?: EmbeddingProvider; - - /** - * LLM provider used by ask() and askStream(). - * Required to call those methods; other methods work without it. - */ - llmProvider?: LlmProvider; -} -``` - -**Throws** `DatabaseError` if the database file cannot be opened or migrations fail. - ---- - -### `index(docs)` - -```ts -async index(docs: LiteDoc[]): Promise<void> -``` - -Index an array of pre-parsed documents. Each document is chunked using the markdown-aware chunker (or code-aware tree-sitter chunker when `language` is set), embedded, and stored. - -**`LiteDoc`** - -```ts -interface LiteDoc { - /** Document title. Required. Used in search result display and title boosting. */ - title: string; - - /** Full document text. Required. Will be chunked before embedding. */ - content: string; - - /** Source URL. Used for deduplication: if a document with this URL exists, - * it is replaced if the content hash changed, skipped if unchanged. */ - url?: string; - - /** - * Source type for provenance tracking. - * @default "manual" - */ - sourceType?: "manual" | "library" | "topic" | "model-generated"; - - /** Library namespace. Allows scoping search to a specific library. */ - library?: string; - - /** Library version. Used with library for version-scoped search. */ - version?: string; - - /** Topic ID to associate the document with for topic-scoped search. */ - topicId?: string; - - /** - * Language alias for code-aware tree-sitter chunking. - * When set and tree-sitter is available, chunks at function/class boundaries - * instead of text boundaries. Falls back silently to the standard text chunker - * if tree-sitter is not installed or parsing fails. - * - * Supported languages and aliases: - * - `"typescript"` (aliases: `"ts"`, `"tsx"`) - * - `"javascript"` (aliases: `"js"`, `"jsx"`, `"mjs"`, `"cjs"`) - * - `"python"` (alias: `"py"`) - * - `"csharp"` (alias: `"cs"`) - * - `"cpp"` (aliases: `"cc"`, `"cxx"`, `"hpp"`, `"h"`) - * - `"c"` - * - `"go"` - */ - language?: string; + dbPath?: string; // SQLite file or ":memory:" (give dbPath or db) + db?: Database.Database; // open better-sqlite3 database; close() leaves it open + provider?: EmbeddingProvider; // default: local all-MiniLM-L6-v2 + llmProvider?: LlmProvider; // LLM for ask() and askStream() + chunker?: Chunker; // custom chunker, e.g. createCodeChunker() + config?: ConfigOverrides; // config values over the defaults } ``` -**`LiteDoc` properties:** - -| Property | Type | Required | Description | -|---|---|---|---| -| `title` | `string` | Yes | Document title. Used in search result display and title boosting. | -| `content` | `string` | Yes | Full document text. Will be chunked before embedding. | -| `url` | `string` | No | Source URL for deduplication — replaced if content hash changed, skipped if unchanged. | -| `sourceType` | `string` | No | `"manual"` (default), `"library"`, `"topic"`, or `"model-generated"`. | -| `library` | `string` | No | Library namespace for scoped search. | -| `version` | `string` | No | Library version. Used with `library` for version-scoped search. | -| `topicId` | `string` | No | Topic ID to associate the document with for topic-scoped search. | -| `language` | `string` | No | Language alias for code-aware tree-sitter chunking (e.g. `"typescript"`, `"cpp"`, `"go"`). When set and tree-sitter is available, chunks at function/class boundaries instead of text boundaries. | - -**Example:** - -```ts -await lite.index([ - { - title: "Rate Limiting", - content: "Apply rate limiting using the X-RateLimit-* headers...", - library: "api", - version: "3.2", - url: "https://docs.example.com/rate-limiting", - }, -]); -``` - ---- - -### `indexRaw(input)` - -```ts -async indexRaw(input: RawInput): Promise<string> -``` - -Index from a raw input source. The input is passed through the parser pipeline (same parsers as the CLI `add` command), normalized to markdown, then chunked and indexed. - -Returns the document ID of the newly created document. - -**`RawInput`** - -```ts -type RawInput = - | { type: "file"; path: string; title?: string } - | { type: "url"; url: string; title?: string } - | { type: "text"; content: string; title: string } - | { type: "buffer"; buffer: Buffer; filename: string; title?: string }; -``` - -| `type` | Description | Format detection | -|---|---|---| -| `"file"` | Read from local filesystem | File extension (`.md`, `.pdf`, `.docx`, etc.) | -| `"url"` | Fetch and parse a web page | Content-Type header | -| `"text"` | Plain text or markdown string | Always treated as markdown | -| `"buffer"` | In-memory buffer (e.g., upload) | `filename` extension | - -**Supported formats:** Markdown, plain text, HTML, PDF (`pdf-parse`), DOCX (`mammoth`), EPUB (`epub2`), PPTX (`pizzip`), CSV, JSON, YAML. - -**Example:** - -```ts -const id1 = await lite.indexRaw({ type: "file", path: "./README.md" }); -const id2 = await lite.indexRaw({ type: "url", url: "https://docs.example.com" }); -const id3 = await lite.indexRaw({ type: "text", title: "Notes", content: "# My Notes\n..." }); -``` - ---- - -### `indexBatch(docs, opts)` +Returns `LibScope.create({ ...options, useConfigFile: false })`: config files, `secrets.json` and `LIBSCOPE_*` config variables are not read. -```ts -async indexBatch(docs: LiteDoc[], opts: { concurrency: number }): Promise<void> -``` +**Throws** `ValidationError` when neither `dbPath` nor `db` is given. Throws `ConfigError` when the database was built with a different embedding model. -Index multiple documents with concurrency control. Documents are embedded in parallel up to `concurrency` at a time. Each document's database write is still atomic. +### Main methods -| Parameter | Type | Description | -|---|---|---| -| `docs` | `LiteDoc[]` | Documents to index | -| `opts.concurrency` | `number` | Max parallel embedding calls. Recommended: 4–8. | +| Method | Result | +| ------------------------------ | ---------------------------------------------------------------------------------------------------------- | +| `add(input \| source)` | `{ kind, documents: [{ documentId, title, chunkCount, source }], errors, skipped }` | +| `search(input \| query)` | `{ items: SearchResult[], total, limit, offset }` | +| `ask(input \| question)` | `{ mode: "answer", answer, sources, model, tokensUsed? }` or `{ mode: "context", contextPrompt, sources }` | +| `askStream(input \| question)` | Async iterator of `{ token }`, then `{ done: true, sources, model }` | +| `overview()` | `{ stats, topics, packs, index, health }` | +| `close()` | Closes the database (unless it was passed as `db`) | -**Example:** +`SearchResult` has `documentId`, `chunkId`, `title`, `content`, `score`, `sourceType`, `library`, `version`, `topicId`, `url`, `avgRating` and `scoreExplanation`. -```ts -await lite.indexBatch( - files.map((f) => ({ title: f.name, content: f.text, library: "docs" })), - { concurrency: 6 }, -); -``` +`docs`, `topics`, `tags`, `links`, `searches`, `packs`, `registries`, `connectors`, `tasks`, `admin`, `analytics` and `webhooks` have the same methods as on `LibScope.create()`. --- -### `search(query, opts?)` +## `createCodeChunker(options?)` ```ts -async search(query: string, opts?: LiteSearchOptions): Promise<LiteSearchResult[]> +function createCodeChunker(options?: { language?: string }): Chunker; ``` -Hybrid vector + FTS5 search using Reciprocal Rank Fusion. Returns chunks ranked by relevance. - -**`LiteSearchOptions`** - -```ts -interface LiteSearchOptions { - /** Maximum number of results. Default: 10. Max: 1000. */ - limit?: number; - - /** Restrict results to a specific library namespace. */ - library?: string; - - /** Restrict results to documents in this topic. */ - topic?: string; - - /** Restrict to documents with all of these tags. */ - tags?: string[]; - - /** - * MMR diversity reranking coefficient (0–1). - * 0 = pure relevance order. 1 = maximum diversity (no two similar chunks). - * Default: no reranking. - */ - diversity?: number; -} -``` +Returns a `Chunker` that splits code with tree-sitter. The language is `options.language`, else the extension of the document's source path or URL, else the extension of its title. It returns `undefined` (use the built-in chunker) for unsupported languages, and when tree-sitter is not installed or cannot parse the code. -**`LiteSearchResult`** +## `Chunker` ```ts -interface LiteSearchResult { - /** Document ID. Use with rate() to record feedback. */ - docId: string; - - /** Chunk ID within the document. */ - chunkId: string; - - /** Document title. */ - title: string; - - /** Chunk text (the actual content that matched). */ +type Chunker = (doc: { content: string; - - /** - * Relevance score. Higher is better. - * Combines vector similarity, BM25, and title boost. - */ - score: number; - - /** Source URL if set at index time, otherwise null. */ - url: string | null; -} -``` - -**Example:** - -```ts -const results = await lite.search("JWT token validation", { - limit: 5, - library: "auth-service", - diversity: 0.2, -}); - -for (const r of results) { - console.log(`[${r.score.toFixed(3)}] ${r.title}`); - console.log(r.content.slice(0, 200)); -} -``` - ---- - -### `getContext(question, opts?)` - -```ts -async getContext(question: string, opts?: LiteContextOptions): Promise<string> -``` - -Retrieve top-K relevant chunks and return them as a formatted context string. Does not call an LLM — returns the context ready for injection into an external prompt. - -This is the primary integration point for external LLM pipelines. - -**`LiteContextOptions`** - -```ts -interface LiteContextOptions { - /** Number of chunks to retrieve. Default: 5. */ - topK?: number; - - /** Restrict retrieval to a specific library. */ - library?: string; - - /** Restrict retrieval to a specific topic. */ - topic?: string; -} -``` - -**Returns:** A formatted string containing the retrieved chunks with their titles. The exact format is: - -``` -[Document Title] -Chunk text here... - -[Another Document] -More chunk text... -``` - -**Example:** - -```ts -const context = await lite.getContext("How do I handle auth errors?", { topK: 3 }); -const prompt = `You are a helpful assistant. Answer using only this context: - -${context} - -Question: How do I handle auth errors?`; -``` - ---- - -### `ask(question, opts?)` - -```ts -async ask(question: string, opts?: LiteAskOptions): Promise<string> -``` - -Full RAG: retrieves context then calls an LLM to produce a grounded answer. - -Requires an `llmProvider` configured in the constructor or passed in `opts`. - -**`LiteAskOptions`** - -```ts -interface LiteAskOptions { - /** Number of context chunks to retrieve. Default: 5. */ - topK?: number; - - /** Scope retrieval to a library. */ - library?: string; - - /** Scope retrieval to a topic. */ - topic?: string; - - /** Custom system prompt. Overrides the default "answer using context" instruction. */ - systemPrompt?: string; - - /** - * LLM provider for this request. - * Overrides the instance-level llmProvider for this single call. - */ - llmProvider?: LlmProvider; -} -``` - -**Returns:** The LLM's answer as a plain string. - -**Throws** `Error` if no `llmProvider` is configured. - -**Example:** - -```ts -const answer = await lite.ask("What authentication methods does the API support?", { - library: "api-docs", - topK: 8, - systemPrompt: "You are a concise technical assistant. Answer in bullet points.", -}); -``` - ---- - -### `askStream(question, opts?)` - -```ts -async *askStream(question: string, opts?: LiteAskOptions): AsyncGenerator<string> -``` - -Streaming version of `ask()`. Yields string tokens as they arrive from the LLM. - -Requires an `llmProvider` with a `completeStream()` method. - -**Throws:** -- `Error` if no `llmProvider` is configured -- `Error` if the provider does not support streaming - -**Example:** - -```ts -process.stdout.write("Answer: "); -for await (const token of lite.askStream("Explain the rate limiting algorithm")) { - process.stdout.write(token); -} -process.stdout.write("\n"); -``` - ---- - -### `deleteByLibrary(library)` - -```ts -deleteByLibrary(library: string): void -``` - -Delete all documents and chunks belonging to the given library namespace. Loops internally until all matching documents are removed (batch size is capped at 1000 per iteration). - -| Parameter | Type | Description | -|---|---|---| -| `library` | `string` | Library name to clear — matches the `library` field set at index time | - -Use this before a full reindex to avoid stale content accumulating: - -```ts -// Wipe the old index for this repo, then rebuild -lite.deleteByLibrary("my-repo"); -await lite.indexBatch(freshDocs, { concurrency: 4 }); -``` - ---- - -### `rate(docId, score)` - -```ts -rate(docId: string, score: number): void -``` - -Record a quality rating for a document. Ratings are stored persistently and influence subsequent search rankings — highly-rated documents are boosted. - -| Parameter | Type | Description | -|---|---|---| -| `docId` | `string` | Document ID (from `LiteSearchResult.docId`) | -| `score` | `number` | Rating 1–5 (1 = poor, 5 = excellent) | - -**Throws** `ValidationError` for invalid scores or unknown document IDs. - -**Example:** - -```ts -const results = await lite.search("error handling patterns"); -if (results[0]) { - lite.rate(results[0].docId, 4); // this result was useful -} -``` - ---- - -### `close()` - -```ts -close(): void + title: string; + source: string; // file path or URL; "" for inline content +}) => Promise<string[] | undefined> | string[] | undefined; ``` -Close the database connection and release all resources. Must be called when the `LibScopeLite` instance is no longer needed. - -After `close()`, all other methods will throw if called. +Used by `add` for inline content and local files. Return the chunk texts, or `undefined` to use the built-in markdown chunker. --- ## `TreeSitterChunker` -Code-aware chunker using tree-sitter AST parsing. Optional — requires `tree-sitter` and at least one grammar package. +Code-aware chunker using tree-sitter AST parsing. Requires `tree-sitter` and at least one grammar package. The parser and grammars load on the first `chunk()` call and are cached. Create one instance and reuse it. -### Constructor +### `supports(language)` ```ts -new TreeSitterChunker() +supports(language: string): boolean ``` -The parser and grammar instances are lazily initialized on first `chunk()` call and cached for the lifetime of the instance. Create one `TreeSitterChunker` and reuse it across all files. - ---- +Returns `true` if the language name or alias is supported (case-insensitive): `typescript`/`ts`/`tsx`, `javascript`/`js`/`jsx`/`mjs`/`cjs`, `python`/`py`, `csharp`/`cs`, `cpp`/`cc`/`cxx`/`hpp`/`h`, `c`, `go`. It does not throw. -### `supports(language)` +### `chunk(source, language, maxChunkSize?)` ```ts -supports(language: string): boolean +async chunk(source: string, language: string, maxChunkSize?: number): Promise<CodeChunk[]> ``` -Returns `true` if the given language alias is supported. Case-insensitive. +| Parameter | Type | Default | Description | +| -------------- | -------- | ------- | ---------------------------- | +| `source` | `string` | — | Source code to chunk | +| `language` | `string` | — | Language name or alias | +| `maxChunkSize` | `number` | `1500` | Maximum characters per chunk | ```ts -chunker.supports("ts"); // true -chunker.supports("TypeScript"); // true -chunker.supports("go"); // true -chunker.supports("cs"); // true -chunker.supports("rust"); // false (not supported) -chunker.supports("unknown"); // false +interface CodeChunk { + content: string; // source text of the chunk + startLine: number; // 1-based + endLine: number; // 1-based + nodeType: string; // e.g. "function_declaration", "class_declaration", "preamble", "module" +} ``` -Does not throw. Safe to call before attempting `chunk()`. +**Throws** `ValidationError` when the language is not supported, tree-sitter is not installed, or the source cannot be parsed. --- -### `chunk(source, language, maxChunkSize?)` +## `normalizeRawInput(input)` ```ts -async chunk( - source: string, - language: string, - maxChunkSize?: number, -): Promise<CodeChunk[]> -``` - -Parse `source` and return an array of semantically meaningful chunks. +function normalizeRawInput(input: RawInput): Promise<NormalizedInput>; -| Parameter | Type | Default | Description | -|---|---|---|---| -| `source` | `string` | — | Source code to chunk | -| `language` | `string` | — | Language name or alias (e.g., `"ts"`, `"python"`) | -| `maxChunkSize` | `number` | `1500` | Maximum characters per chunk | - -**`CodeChunk`** +type RawInput = + | { type: "file"; path: string; title?: string } + | { type: "url"; url: string; title?: string } + | { type: "text"; content: string; title: string } + | { type: "buffer"; buffer: Buffer; filename: string; title?: string }; -```ts -interface CodeChunk { - /** Source text of this chunk (function body, class, etc.) */ +interface NormalizedInput { + title: string; content: string; - - /** 1-based line number where this chunk starts in the original source. */ - startLine: number; - - /** 1-based line number where this chunk ends. */ - endLine: number; - - /** - * Tree-sitter node type. Common values: - * - "function_declaration" - * - "class_declaration" - * - "method_definition" - * - "export_statement" - * - "preamble" (accumulated imports/comments before first declaration) - * - "trailing" (non-declaration nodes after last declaration) - * - "module" (entire source, returned when no declarations found) - */ - nodeType: string; } ``` -**Throws** `ValidationError`: -- If `language` is not in the supported list -- If `tree-sitter` is not installed (with install instructions) -- If the source file cannot be parsed - -**Example:** - -```ts -const chunks = await chunker.chunk( - await readFile("src/api.ts", "utf8"), - "typescript", - 2000, -); - -console.log(`${chunks.length} chunks`); -chunks.forEach((c) => { - console.log(` ${c.nodeType} (lines ${c.startLine}–${c.endLine}): ${c.content.length} chars`); -}); -``` +Files and buffers are parsed by the parser for their extension (PDF, DOCX, PPTX, EPUB, HTML, CSV, YAML, JSON, Markdown, text). Other files, for example source code, are read as UTF-8 text. The default title is the file name without its extension. URLs are fetched and converted to markdown. --- -## Type Reference - -### `EmbeddingProvider` +## Provider interfaces ```ts interface EmbeddingProvider { @@ -593,60 +153,37 @@ interface EmbeddingProvider { embed(text: string): Promise<number[]>; embedBatch(texts: string[]): Promise<number[][]>; } -``` - -Import from `libscope`: -```ts -import type { EmbeddingProvider } from "libscope"; -``` - -### `LlmProvider` - -```ts interface LlmProvider { - model: string; - complete(prompt: string, systemPrompt?: string): Promise<{ text: string }>; - completeStream?(prompt: string, systemPrompt?: string): AsyncGenerator<string>; + readonly model: string; + complete(prompt: string, systemPrompt?: string): Promise<{ text: string; tokensUsed?: number }>; + completeStream?(prompt: string, systemPrompt?: string): AsyncIterable<string>; } ``` -Import from `libscope`: - -```ts -import type { LlmProvider } from "libscope"; -``` - --- -## Error Types - -All errors extend `LibScopeError` with a `.code` string property: +## Errors -| Class | Code | When thrown | -|---|---|---| -| `DatabaseError` | `DATABASE_ERROR` | SQLite failures, schema errors | -| `ValidationError` | `VALIDATION_ERROR` | Bad input, unsupported language, missing tree-sitter | -| `EmbeddingError` | `EMBEDDING_ERROR` | Embedding provider failures | -| `DocumentNotFoundError` | `DOCUMENT_NOT_FOUND` | `rate()` with unknown docId | +All errors extend `LibScopeError` and have a `code`: -```ts -import { ValidationError, DatabaseError } from "libscope"; - -try { - await lite.index([{ title: "", content: "..." }]); -} catch (err) { - if (err instanceof ValidationError) { - console.error("Invalid input:", err.message); // "Document title is required" - } -} -``` +| Class | Code | When | +| ----------------------- | ------------------------------ | ----------------------------------------------------------- | +| `ValidationError` | `VALIDATION_ERROR` | Invalid input, unsupported language, tree-sitter missing | +| `ConfigError` | `CONFIG_ERROR` | No LLM for `ask`, missing API key, embedding model mismatch | +| `NotFoundError` | `NOT_FOUND` or a specific code | Unknown link, webhook, saved search, version, pack, task | +| `DocumentNotFoundError` | `DOCUMENT_NOT_FOUND` | Unknown `documentId` | +| `ChunkNotFoundError` | `CHUNK_NOT_FOUND` | Unknown `chunkId` | +| `TopicNotFoundError` | `TOPIC_NOT_FOUND` | Unknown topic | +| `DatabaseError` | `DATABASE_ERROR` | SQLite failures | +| `EmbeddingError` | `EMBEDDING_ERROR` | Embedding provider failures | +| `FetchError` | `FETCH_ERROR` | URL fetch failures | --- -## See Also +## See also -- [LibScope Lite Guide](/guide/lite) — usage guide with examples -- [Code Indexing Guide](/guide/code-indexing) — tree-sitter chunking in depth -- [How Search Works](/guide/how-search-works) — hybrid vector + FTS5 explained -- [Configuration Reference](/reference/configuration) — embedding providers, LLM setup +- [LibScope Lite Guide](/guide/lite) +- [Programmatic Usage](/guide/programmatic-usage) +- [Code Indexing Guide](/guide/code-indexing) +- [Configuration Reference](/reference/configuration) diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 55db89b8..ba823d7d 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -1,294 +1,297 @@ # MCP Tools -LibScope exposes the following tools over the Model Context Protocol. Any MCP-compatible client (Claude, Cursor, VS Code, etc.) can call these directly. +LibScope exposes its knowledge base over the Model Context Protocol. Any MCP client (Claude, Cursor, VS Code, and others) can call these tools. See [MCP Setup](/guide/mcp-setup) to connect a client. -## search-docs +The server has 11 core tools. Four admin tools are available when you enable the admin toolset. Each tool is generated from a LibScope operation, so its parameters, defaults and validation are the same as in the CLI, the REST API and the SDK. -Semantic search across your knowledge base. +## Workflow -| Parameter | Type | Required | Description | -| ----------- | ------ | -------- | ---------------------------- | -| `query` | string | ✅ | Search query | -| `library` | string | | Filter by library name | -| `topic` | string | | Filter by topic | -| `version` | string | | Filter by library version | -| `minRating` | number | | Minimum average rating (1–5) | -| `limit` | number | | Max results (default: 10) | -| `offset` | number | | Pagination offset | +1. `search` finds chunks. Each result shows a `documentId` and a `chunkId`. +2. `get-document` reads a document with its tags, links and ratings. For a long document, set `maxLength`, then pass the `offset` that the output shows as "next page". +3. `rate-document` records whether a document was useful, wrong or out of date. -**Search results** include a `scoreExplanation` object on each result: +The server sends these steps to the client as its `instructions`, so the assistant knows the workflow without extra prompting. -```typescript -{ - method: "hybrid" | "vector" | "fts5" | "keyword", - rawScore: number, // raw score before boosts - boostFactors: string[], // e.g. ["title_match:x1.5"] - details: string // human-readable scoring breakdown -} -``` +## Output + +Tools return compact text. Each line that names a document, chunk, link or task includes its ID (`documentId`, `chunkId`, `linkId`, `taskId`), so the assistant can pass the ID to the next tool. Lists show their position, for example `Documents 1-50 of 120 — next page: offset 50`. -## get-document +Errors return `isError: true` with a message. Invalid parameters fail with an input validation error that names the parameter. -Retrieve a document by its ID, including ratings and metadata. +## Background tasks -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | --------------- | -| `documentId` | string | ✅ | The document ID | +`submit-document`, `sync`, `install-pack` and `reindex-documents` accept `async: true`. With it, the tool starts a background task and returns a `taskId` immediately. Then: -## delete-document +- `task {"action": "status", "taskId": "..."}` shows the status (`pending`, `running`, `completed`, `failed` or `cancelled`), the progress and, when the task is done, the same output that the tool returns without `async`. +- `task {"action": "cancel", "taskId": "..."}` asks the task to stop. +- `task {"action": "list"}` lists the tasks from the last hour. -Delete a document from the knowledge base. +Tasks are kept in memory for one hour after they finish. They are lost when the server stops. -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | ------------------------- | -| `documentId` | string | ✅ | The document ID to delete | +## ask and passthrough -## submit-document +`ask` is registered only when it can answer: -Index a new document. You can provide content directly, or a URL to fetch automatically. +- With `llm.provider` set to `auto` (the default) or `passthrough`, the MCP server does not call an LLM. `ask` returns the retrieved context and its sources. The calling assistant writes the answer. +- With `llm.provider` set to `openai`, `anthropic` or `ollama` and the provider configured, `ask` returns the LLM's answer and its sources. +- When an LLM provider is selected but cannot be created (for example, an API key is missing), the server does not register `ask`. -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | ---------------------------------------------------- | -| `title` | string | | Document title (auto-detected from URL if omitted) | -| `content` | string | | Document content in markdown (omit if providing URL) | -| `url` | string | | URL to fetch and index | -| `library` | string | | Library name | -| `version` | string | | Library version | -| `topic` | string | | Topic to categorize under | -| `sourceType` | string | | `library`, `topic`, `manual`, or `model-generated` | -| `dedup` | string | | Duplicate detection behaviour (see below) | -| `dedupOptions` | object | | Fine-tune duplicate detection (see below) | +## Admin toolset -**`dedup`** *(optional)*: Controls duplicate detection behaviour. -- `"skip"` — If a duplicate is detected, return the existing document without re-indexing -- `"warn"` — Log a warning about the duplicate but index anyway -- `"force"` — Skip duplicate checking entirely and always index -- *(omitted)* — Default behaviour: reject exact duplicates by title+content-length, allow similar content +The `mcp.toolsets` config key enables optional toolsets (`libscope config set mcp.toolsets admin`, or the environment variable `LIBSCOPE_MCP_TOOLSETS` as a comma-separated list). `admin` enables `sync`, `install-pack`, `list-packs` and `reindex-documents`. `all` enables every optional toolset. `core` (the default tools) is always on. + +```json +{ + "mcpServers": { + "libscope": { + "command": "npx", + "args": ["-y", "libscope", "serve"], + "env": { "LIBSCOPE_MCP_TOOLSETS": "admin" } + } + } +} +``` -**`dedupOptions`** *(optional)*: Fine-tune duplicate detection. -- `threshold` *(number, 0–1)*: Similarity threshold for semantic dedup (default 0.95) -- `strategy` *(string)*: `"exact"` (hash-based) or `"semantic"` (embedding-based) +`sync` uses connections saved with `libscope connect`. It does not accept tokens or other credentials as parameters. -## update-document +## Core tools -Update an existing document's title, content, or metadata. Changing content triggers re-chunking and re-embedding. +The tool descriptions and parameter tables below are generated from the server code (`npm run docs:gen`). -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | --------------------------------------- | -| `documentId` | string | ✅ | The document ID to update | -| `title` | string | | New title | -| `content` | string | | New content (triggers re-chunking) | -| `library` | string | | New library name (pass `null` to clear) | -| `version` | string | | New version (pass `null` to clear) | -| `url` | string | | New source URL (pass `null` to clear) | -| `topicId` | string | | New topic ID (pass `null` to clear) | +<!-- generated:start mcp:core --> -## get-related +### search -Find document chunks that are semantically similar to a given chunk (more-like-this). +Search the knowledge base by meaning and keywords (query), or find content similar to a document or chunk (relatedTo). Results carry documentId and chunkId. -| Parameter | Type | Required | Description | -| ----------- | ------ | -------- | ---------------------------------- | -| `chunkId` | string | ✅ | The source chunk ID | -| `limit` | number | | Max results (default: 10) | -| `library` | string | | Filter results by library | -| `topic` | string | | Filter results by topic | +Annotations: read-only. -## rate-document +| Parameter | Type | Required | Description | +| ---------------------- | ----------------------------------------------------- | -------- | --------------------------------------------------------------------------- | +| `query` | string | | What to search for | +| `relatedTo` | string | | Document or chunk ID: return similar content instead of running a query | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | +| `tags` | string[] | | Only documents carrying all of these tags | +| `minRating` | number | | Minimum average rating | +| `limit` | integer | | Maximum results (default 10, max 100) | +| `offset` | integer | | Results to skip (paging). Default: `0`. | +| `maxChunksPerDocument` | integer | | At most this many chunks per document (default: no limit) | +| `contextChunks` | integer | | Neighbouring chunks to include before and after each result. Default: `0`. | +| `diversity` | number | | Query search: MMR reranking, 0 = relevance only (default), 1 = most diverse | -Rate a document and optionally suggest corrections. +### ask -| Parameter | Type | Required | Description | -| --------------------- | ------ | -------- | ------------------------------------- | -| `documentId` | string | ✅ | The document ID | -| `rating` | number | ✅ | Rating from 1 (poor) to 5 (excellent) | -| `chunkId` | string | | Rate a specific chunk | -| `feedback` | string | | Text feedback | -| `suggestedCorrection` | string | | Suggested replacement content | +Answer a question from the knowledge base with an LLM, or (passthrough) return the context to answer from -## list-documents +Annotations: read-only. -List documents with optional filters. +| Parameter | Type | Required | Description | +| -------------- | ----------------------------------------------------- | -------- | ---------------------------------------------------------------------------- | +| `question` | string | yes | The question | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | +| `tags` | string[] | | Only documents carrying all of these tags | +| `minRating` | number | | Minimum average rating | +| `topK` | integer | | Chunks to retrieve as context. Default: `5`. | +| `systemPrompt` | string | | System prompt for the LLM (default: answer from the context and cite titles) | -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | ------------------------- | -| `library` | string | | Filter by library | -| `topic` | string | | Filter by topic | -| `sourceType` | string | | Filter by source type | -| `limit` | number | | Max results (default: 50) | +### get-document -## list-topics +Get a document with its tags, links and rating summary; long content can be paged -List available topics. +Annotations: read-only. -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | -------------------------------------- | -| `parentId` | string | | Filter by parent topic (for subtopics) | +| Parameter | Type | Required | Description | +| ------------ | ------- | -------- | ------------------------------------------------------ | +| `documentId` | string | yes | Document ID | +| `offset` | integer | | Character offset into the content. Default: `0`. | +| `maxLength` | integer | | Maximum characters of content to return (default: all) | -## ask-question +### list-documents -RAG question-answering. Retrieves relevant chunks and synthesizes an answer using your configured LLM. +List documents (newest first) with optional filters -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | ----------------------------------------- | -| `question` | string | ✅ | The question to answer | -| `library` | string | | Filter source docs by library | -| `topic` | string | | Filter source docs by topic | -| `topK` | number | | Number of chunks to retrieve (default: 5) | +Annotations: read-only. -## health-check +| Parameter | Type | Required | Description | +| ------------ | ----------------------------------------------------- | -------- | ----------------------------------------- | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Document source type | +| `tags` | string[] | | Only documents carrying all of these tags | +| `limit` | integer | | Maximum results (default 50, max 1000) | +| `offset` | integer | | Results to skip (paging). Default: `0`. | -Check database connectivity, document/chunk counts, and FTS5 index status. Takes no parameters. +### overview -## reindex-documents +Knowledge base overview: counts, topics, installed packs, embedding model of the index, and health -Re-embed all document chunks with the current embedding provider. Use after switching providers. +Annotations: read-only. -| Parameter | Type | Required | Description | -| ------------- | -------- | -------- | --------------------------------------------------- | -| `batchSize` | number | | Chunks per batch (default: 50) | -| `since` | string | | Only reindex docs created after this ISO-8601 date | -| `before` | string | | Only reindex docs created before this ISO-8601 date | -| `documentIds` | string[] | | Only reindex specific documents | +No parameters. -## sync-obsidian-vault +### submit-document -Sync an Obsidian vault into the knowledge base. Parses wikilinks, frontmatter, embeds, and tags. +Add to the knowledge base: inline content (with title), a web page (url), a site crawl (url + spider: true) or a public GitHub/GitLab repository URL. Local file paths are not accepted. -| Parameter | Type | Required | Description | -| ----------- | ------ | -------- | ------------------------------------ | -| `vaultPath` | string | ✅ | Absolute path to the vault directory | +Annotations: not destructive. -## sync-notion +| Parameter | Type | Required | Description | +| ----------------- | ----------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------- | +| `content` | string | | Inline document content (markdown) | +| `title` | string | | Title (required with content; detected for files and URLs) | +| `url` | string (uri) | | Source URL. Without content, the URL is fetched; with content, it is stored | +| `topic` | string | | Topic ID or name | +| `library` | string | | Library name | +| `version` | string | | Library version | +| `sourceType` | `library` \| `topic` \| `manual` \| `model-generated` | | Source type (default: library if library is set, topic if topic is set, else manual) | +| `tags` | string[] | | Tags to add to every new document | +| `expiresAt` | string (date-time) | | ISO 8601 time after which the document is pruned | +| `dedup` | `skip` \| `warn` \| `force` | | Duplicate handling: skip returns the existing document, warn indexes anyway, force skips the check | +| `spider` | boolean | | URL: also crawl linked pages. Default: `false`. | +| `maxPages` | integer | | Crawl: page limit (default 25, max 200) | +| `maxDepth` | integer | | Crawl: link depth (default 2, max 5) | +| `sameDomain` | boolean | | Crawl: stay on the seed domain (default true) | +| `pathPrefix` | string | | Crawl: only follow links under this path | +| `excludePatterns` | string[] | | Crawl: globs of URLs to skip | +| `branch` | string | | Repository: branch (default: from URL, else main) | +| `paths` | string[] | | Repository: only these subdirectories | +| `extensions` | string[] | | Repository: file extensions (default .md, .mdx, .txt, .rst) | +| `dryRun` | boolean | | List what would be added without adding it. Default: `false`. | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"}. Default: `false`. | -Sync Notion pages and databases. +### update-document -| Parameter | Type | Required | Description | -| -------------- | -------- | -------- | --------------------------------------- | -| `token` | string | ✅ | Notion integration token | -| `excludePages` | string[] | | Page/database IDs to exclude | -| `lastSync` | string | | ISO-8601 timestamp for incremental sync | +Update a document's title, content, metadata or tags (content changes are re-indexed) -## sync-confluence +Annotations: not destructive, idempotent. -Sync Confluence spaces and pages. +| Parameter | Type | Required | Description | +| ------------ | -------------------- | -------- | ---------------------------------------- | +| `documentId` | string | yes | Document ID | +| `title` | string | | New title | +| `content` | string | | New content (re-chunked and re-embedded) | +| `library` | string \| null | | New library (null clears it) | +| `version` | string \| null | | New version (null clears it) | +| `url` | string (uri) \| null | | New URL (null clears it) | +| `topic` | string \| null | | New topic ID or name (null clears it) | +| `tags` | string[] | | Replace the document's tags with these | -| Parameter | Type | Required | Description | -| --------------- | -------- | -------- | --------------------------------- | -| `baseUrl` | string | ✅ | Confluence base URL | -| `email` | string | ✅ | User email | -| `token` | string | ✅ | API token | -| `spaces` | string[] | | Space keys to sync (default: all) | -| `excludeSpaces` | string[] | | Space keys to exclude | +### delete-document -## sync-slack +Delete a document with its chunks, vectors, tags, links and ratings -Sync Slack channel messages and threads. +Annotations: destructive. -| Parameter | Type | Required | Description | -| ----------------- | -------- | -------- | ----------------------------------- | -| `token` | string | ✅ | Slack bot token | -| `channels` | string[] | ✅ | Channel names/IDs, or `["all"]` | -| `excludeChannels` | string[] | | Channels to exclude | -| `threadMode` | string | | `aggregate` (default) or `separate` | +| Parameter | Type | Required | Description | +| ------------ | ------ | -------- | ----------- | +| `documentId` | string | yes | Document ID | -## sync-onenote +### rate-document -Sync OneNote notebooks via Microsoft Graph API. +Rate a document (1-5), optionally with feedback or a suggested correction -| Parameter | Type | Required | Description | -| -------------- | ------ | -------- | -------------------------------- | -| `accessToken` | string | ✅ | Microsoft Graph API access token | -| `notebookName` | string | | Specific notebook (default: all) | +Annotations: not destructive. -## install-pack +| Parameter | Type | Required | Description | +| --------------------- | ------- | -------- | ---------------------------------------- | +| `documentId` | string | yes | Document ID | +| `chunkId` | string | | Rate one chunk of the document | +| `rating` | integer | yes | 1 (poor) to 5 (excellent) | +| `feedback` | string | | What is good or wrong | +| `suggestedCorrection` | string | | Replacement text if the content is wrong | -Install a knowledge pack from the registry or a local file. +### link-documents -| Parameter | Type | Required | Description | -| ------------- | ------ | -------- | ---------------------------- | -| `nameOrPath` | string | ✅ | Pack name or local file path | -| `registryUrl` | string | | Custom registry URL | +Create a typed link from one document to another (action: create), or delete a link by linkId (action: delete). get-document lists a document's links. -## list-packs +Annotations: destructive. -List installed or available knowledge packs. +| Parameter | Type | Required | Description | +| ------------------ | ------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------------------------------- | +| `action` | `create` \| `delete` | yes | What to do: create, delete | +| `documentId` | string | | Source document ID (action: create) | +| `targetDocumentId` | string | | Target document ID (action: create) | +| `linkType` | `see_also` \| `prerequisite` \| `supersedes` \| `related` \| `references` | | Relationship type: see_also, prerequisite, supersedes, related, references (action: create) | +| `label` | string | | Short description of the relationship (action: create) | +| `linkId` | string | | Link ID (action: delete) | -| Parameter | Type | Required | Description | -| ------------- | ------- | -------- | ------------------------------------------------ | -| `available` | boolean | | If true, list from registry instead of installed | -| `registryUrl` | string | | Custom registry URL | +### task -## suggest-tags +Background tasks started with async: true. status: progress and result of a task; cancel: stop it; list: tasks from the last hour. -Suggest tags for a document based on content analysis (compares against existing tags in the knowledge base). +Annotations: not destructive, idempotent. -| Parameter | Type | Required | Description | -| ---------------- | ------ | -------- | ------------------------------------ | -| `documentId` | string | ✅ | The document ID | -| `maxSuggestions` | number | | Maximum suggestions to return (1–20, default: 5) | +| Parameter | Type | Required | Description | +| --------- | ------------------------------ | -------- | ------------------------------------------------------------------------ | +| `action` | `status` \| `cancel` \| `list` | yes | What to do: status, cancel, list | +| `taskId` | string | | Task ID returned when a background task started (action: status, cancel) | -## link-documents +<!-- generated:end mcp:core --> -Create a typed cross-reference relationship between two documents. +## Admin tools -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | -------------------------------------------------------------------- | -| `sourceId` | string | ✅ | The source document ID | -| `targetId` | string | ✅ | The target document ID | -| `linkType` | string | ✅ | Relationship type: `see_also`, `prerequisite`, `supersedes`, `related` | -| `label` | string | | Optional human-readable description of the relationship | +These tools are registered only when `mcp.toolsets` contains `admin` or `all`. -## get-document-links +<!-- generated:start mcp:admin --> -Get all cross-reference links for a document, both outgoing and incoming. +### sync -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | --------------- | -| `documentId` | string | ✅ | The document ID | +Sync one saved connector connection (name) or all of them (all: true) with the settings saved by 'libscope connect' -## delete-link +Annotations: not destructive. -Remove a cross-reference link between documents. +| Parameter | Type | Required | Description | +| --------- | ------- | -------- | --------------------------------------------------------------------------------------------------------- | +| `name` | string | | Name of a saved connection | +| `all` | boolean | | Sync every saved connection. Default: `false`. | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"}. Default: `false`. | -| Parameter | Type | Required | Description | -| --------- | ------ | -------- | ------------------------ | -| `linkId` | string | ✅ | The link ID to delete | +### install-pack -## save-search +Install a knowledge pack from a registry (name or name@version) -Save a search query with optional filters for later re-use. +Annotations: not destructive, idempotent. -| Parameter | Type | Required | Description | -| ----------- | -------- | -------- | --------------------------------- | -| `name` | string | ✅ | Unique name for this saved search | -| `query` | string | ✅ | The search query | -| `topic` | string | | Filter by topic ID | -| `library` | string | | Filter by library name | -| `version` | string | | Filter by library version | -| `source` | string | | Filter by source type | -| `minRating` | number | | Minimum average rating filter | -| `limit` | number | | Maximum results to return | -| `tags` | string[] | | Filter by tags | +| Parameter | Type | Required | Description | +| ------------- | ------- | -------- | --------------------------------------------------------------------------------------------------------- | +| `pack` | string | yes | Pack name or name@version from a registry, or a local .json/.json.gz file | +| `registry` | string | | Look only in this registry (needed when several registries have the pack) | +| `batchSize` | integer | | Documents per batch (default 10) | +| `concurrency` | integer | | Batches embedded in parallel (default 4) | +| `resumeFrom` | integer | | Skip the first N documents (resume a partial install) | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"}. Default: `false`. | -## list-saved-searches +### list-packs -List all saved searches. +List installed packs, or the packs available in the configured registries -_No parameters._ +Annotations: read-only. -## run-saved-search +| Parameter | Type | Required | Description | +| ----------- | ------- | -------- | ---------------------------------------------------------------------------------------- | +| `available` | boolean | | List the packs in the configured registries instead of installed ones. Default: `false`. | +| `registry` | string | | With available: only this registry | -Execute a saved search by name or ID and return results. +### reindex-documents -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | ----------------------------------------- | -| `nameOrId` | string | ✅ | The name or ID of the saved search to run | +Re-embed chunks with the configured embedding model (rebuild after changing models) -## delete-saved-search +Annotations: not destructive, idempotent. -Delete a saved search by name or ID. +| Parameter | Type | Required | Description | +| ------------- | ----------------------------------- | -------- | --------------------------------------------------------------------------------------------------------- | +| `documentIds` | string[] | | Only these documents | +| `since` | string (date) \| string (date-time) | | Only documents created on or after | +| `before` | string (date) \| string (date-time) | | Only documents created on or before | +| `batchSize` | integer | | Chunks per embedding call (default 50) | +| `rebuild` | boolean | | Drop and recreate the vector table for the configured model, then re-embed everything. Default: `false`. | +| `async` | boolean | | Run in the background and return a taskId at once; poll with task {"action": "status"}. Default: `false`. | -| Parameter | Type | Required | Description | -| ---------- | ------ | -------- | -------------------------------------------- | -| `nameOrId` | string | ✅ | The name or ID of the saved search to delete | +<!-- generated:end mcp:admin --> diff --git a/docs/reference/registry.md b/docs/reference/registry.md index d5b6a568..1cd4cb8d 100644 --- a/docs/reference/registry.md +++ b/docs/reference/registry.md @@ -1,81 +1,79 @@ # Registry Reference -Complete reference for the git-based pack registry feature. +Complete reference for git-based pack registries. A registry is a git repository of knowledge packs; `--registry <name>` always means the name of a configured registry. ## CLI Commands ### `libscope registry add` -Register a git repository as a pack registry. +Add a git repository as a pack registry and clone it. ```bash libscope registry add <git-url> [options] ``` -| Option | Description | -| ----------------------------- | -------------------------------------------------------- | -| `<git-url>` | Git clone URL (HTTPS or SSH) | -| `-n, --name <alias>` | Short name for this registry (default: inferred from URL)| -| `--priority <n>` | Priority for conflict resolution — lower wins (default: 10) | -| `--sync-interval <seconds>` | Auto-sync interval in seconds, 0 = manual only (default: 0) | -| `--no-sync` | Skip the initial sync after adding | +| Option | Description | +| ------------------- | ------------------------------------------------------------ | +| `<git-url>` | Git URL: `https://`, `ssh://`, `git@host:path` or `file:///` | +| `-n, --name <name>` | Registry name (default: the last part of the URL) | +| `--no-sync` | Do not clone the registry now | ```bash # Examples libscope registry add https://github.com/org/registry.git -libscope registry add git@github.com:team/packs.git --name team --priority 5 -libscope registry add https://github.com/org/registry.git --sync-interval 86400 +libscope registry add git@github.com:team/packs.git --name team +libscope registry add file:///srv/libscope-packs.git --no-sync ``` ### `libscope registry remove` -Unregister a registry and delete its local cache. +Remove a registry and delete its local clone. ```bash libscope registry remove <name> [-y, --yes] ``` -| Option | Description | -| ------------ | -------------------------- | -| `-y, --yes` | Skip confirmation prompt | +| Option | Description | +| ----------- | ------------------------ | +| `-y, --yes` | Skip confirmation prompt | ### `libscope registry list` -List all configured registries with their sync status. +List the configured registries. ```bash libscope registry list ``` -Output includes: name, URL, priority, pack count, and last synced timestamp. +Output includes: name, URL, pack count, and last synced timestamp. ### `libscope registry sync` -Manually sync one or all registries (git fetch + fast-forward). +Fetch the latest packs of one or all registries (git fetch + reset to the remote branch). This is the only command that updates the local copies. ```bash libscope registry sync [<name>] ``` -Without a name, syncs all registries. With a name, syncs only that registry. +Without a name, syncs all registries. With a name, syncs only that registry. A registry that cannot be reached keeps its local copy (`offline`); one that was never cloned reports an error and the command exits with code 1. ### `libscope registry search` -Search across cached registry indexes. +Search the local copies of the registry indexes. ```bash -libscope registry search <query> [-r, --registry <name>] +libscope registry search <query> [--registry <name>] ``` -| Option | Description | -| --------------------- | ---------------------------------------------- | -| `<query>` | Search term (matches name, description, tags, author) | -| `-r, --registry <name>` | Limit search to a specific registry | +| Option | Description | +| ------------------- | ----------------------------------------------------- | +| `<query>` | Search term (matches name, description, tags, author) | +| `--registry <name>` | Search only this registry | ```bash # Examples libscope registry search "react" -libscope registry search "kubernetes" -r official +libscope registry search "kubernetes" --registry official ``` ### `libscope registry create` @@ -87,34 +85,37 @@ libscope registry create <path> ``` Creates a git repo with: + - `index.json` — empty pack index (JSON array) - `packs/` — directory for pack contents (with `.gitkeep`) - An initial commit ### `libscope registry publish` -Publish a pack file to a registry. +Publish a pack file (`.json` or `.json.gz`) to a registry. ```bash -libscope registry publish <packFile> -r <name> [options] +libscope registry publish <file> --registry <name> [options] ``` -| Option | Description | -| ----------------------- | -------------------------------------------------------- | -| `<packFile>` | Path to the pack `.json` file to publish | -| `-r, --registry <name>` | Target registry (required) | -| `--version <semver>` | Version to publish as (default: auto-bump patch) | -| `-m, --message <msg>` | Git commit message | -| `--submit` | Push to a feature branch instead of main (for PR workflow) | +| Option | Description | +| ------------------------- | ----------------------------------------------------------------------- | +| `<file>` | Path to the pack `.json` or `.json.gz` file to publish | +| `--registry <name>` | Target registry (required) | +| `--pack-version <semver>` | Version to publish (default: next patch version, or the pack's version) | +| `-m, --message <msg>` | Git commit message | +| `--submit` | Push to a feature branch instead of main (for PR workflow) | **Direct publish** (you have write access): + ```bash -libscope registry publish ./react-docs.json -r my-registry --version 1.0.0 +libscope registry publish ./react-docs.json --registry my-registry --pack-version 1.0.0 ``` **Submit for inclusion** (you don't have write access): + ```bash -libscope registry publish ./react-docs.json -r community --submit +libscope registry publish ./react-docs.json --registry community --submit ``` The `--submit` flag creates a `feature/add-<pack-name>` branch and pushes it. You then create a pull request manually. @@ -124,44 +125,58 @@ The `--submit` flag creates a `feature/add-<pack-name>` branch and pushes it. Yo Remove a specific pack version from a registry. ```bash -libscope registry unpublish <packName> -r <name> --version <semver> [options] +libscope registry unpublish <name>@<version> --registry <name> [options] ``` -| Option | Description | -| ----------------------- | --------------------------------- | -| `<packName>` | Name of the pack to unpublish | -| `-r, --registry <name>` | Target registry (required) | -| `--version <semver>` | Version to remove (required) | -| `-m, --message <msg>` | Git commit message | -| `-y, --yes` | Skip confirmation prompt | +| Option | Description | +| --------------------- | ----------------------------- | +| `<name>@<version>` | Pack and version to unpublish | +| `--registry <name>` | Target registry (required) | +| `-m, --message <msg>` | Git commit message | +| `-y, --yes` | Skip confirmation prompt | If the last version of a pack is unpublished, the entire pack is removed from the registry index. -### `libscope pack install` (extended) +### `libscope pack install` -The existing `pack install` command is extended to resolve packs from registries. +`pack install` resolves pack names from the configured registries. ```bash -libscope pack install <name> [--version <semver>] [--registry <name>] +libscope pack install <name>[@<version>] [--registry <name>] ``` -| Option | Description | -| -------------------- | ---------------------------------------------- | -| `--version <semver>` | Install a specific version (default: latest) | -| `--registry <name>` | Install from a specific registry | +| Option | Description | +| ------------------- | -------------------------------------------------------------- | +| `--registry <name>` | Look only in this registry (needed when several have the pack) | ```bash -# Install latest from any registry +# Install latest from the registry that has the pack libscope pack install react-docs # Install specific version libscope pack install react-docs@1.2.0 -libscope pack install react-docs --version 1.2.0 # Install from a specific registry libscope pack install react-docs --registry official ``` +`libscope pack list --available [--registry <name>]` lists the packs in the registries. + +### Other interfaces + +| Operation | Node.js API | REST | MCP (admin toolset) | +| ------------------- | ------------------------------ | ---------------------------------- | ------------------- | +| `list-registries` | `scope.registries.list()` | `GET /api/v1/registries` | — | +| `search-registries` | `scope.registries.search()` | `GET /api/v1/registries/search` | — | +| `add-registry` | `scope.registries.add()` | — | — | +| `remove-registry` | `scope.registries.remove()` | — | — | +| `sync-registries` | `scope.registries.sync()` | — | — | +| `create-registry` | `scope.registries.create()` | — | — | +| `publish-pack` | `scope.registries.publish()` | — | — | +| `unpublish-pack` | `scope.registries.unpublish()` | — | — | +| `install-pack` | `scope.packs.install()` | `POST /api/v1/packs` | `install-pack` | +| `list-packs` | `scope.packs.list()` | `GET /api/v1/packs?available=true` | `list-packs` | + --- ## Registry Folder Structure @@ -209,14 +224,14 @@ A JSON array of pack summaries for fast search without traversing subdirectories ] ``` -| Field | Type | Description | -| ---------------- | ---------- | ---------------------------------------- | -| `name` | `string` | Pack name (unique within the registry) | -| `description` | `string` | One-line description | -| `tags` | `string[]` | Tags/categories for search filtering | -| `latestVersion` | `string` | Latest published semver version | -| `author` | `string` | Author name or handle | -| `updatedAt` | `string` | ISO-8601 timestamp of last publish | +| Field | Type | Description | +| --------------- | ---------- | -------------------------------------- | +| `name` | `string` | Pack name (unique within the registry) | +| `description` | `string` | One-line description | +| `tags` | `string[]` | Tags/categories for search filtering | +| `latestVersion` | `string` | Latest published semver version | +| `author` | `string` | Author name or handle | +| `updatedAt` | `string` | ISO-8601 timestamp of last publish | ## Schema: `pack.json` @@ -248,18 +263,18 @@ Per-pack manifest with full metadata and version history. } ``` -| Field | Type | Description | -| ------------------------ | ---------- | ---------------------------------------------- | -| `name` | `string` | Pack name | -| `description` | `string` | One-line description | -| `tags` | `string[]` | Tags/categories | -| `author` | `string` | Author name or handle | -| `license` | `string` | License identifier (e.g. "MIT") | -| `versions[].version` | `string` | Semver version string | -| `versions[].publishedAt` | `string` | ISO-8601 publish timestamp | -| `versions[].checksumPath`| `string` | Relative path to the checksum file | -| `versions[].checksum` | `string` | SHA-256 checksum hex value | -| `versions[].docCount` | `number` | Number of documents in this version | +| Field | Type | Description | +| ------------------------- | ---------- | ----------------------------------- | +| `name` | `string` | Pack name | +| `description` | `string` | One-line description | +| `tags` | `string[]` | Tags/categories | +| `author` | `string` | Author name or handle | +| `license` | `string` | License identifier (e.g. "MIT") | +| `versions[].version` | `string` | Semver version string | +| `versions[].publishedAt` | `string` | ISO-8601 publish timestamp | +| `versions[].checksumPath` | `string` | Relative path to the checksum file | +| `versions[].checksum` | `string` | SHA-256 checksum hex value | +| `versions[].docCount` | `number` | Number of documents in this version | Versions are ordered newest first. @@ -275,28 +290,22 @@ Registries are stored in `~/.libscope/config.json` under the `registries` key: { "name": "official", "url": "git@github.com:org/libscope-registry.git", - "syncInterval": 86400, - "priority": 10, "lastSyncedAt": "2026-03-10T14:30:00Z" }, { "name": "team", "url": "https://github.com/team/internal-packs.git", - "syncInterval": 0, - "priority": 5, "lastSyncedAt": null } ] } ``` -| Field | Type | Description | Default | -| -------------- | ----------------- | -------------------------------------------------- | ------- | -| `name` | `string` | Local alias for the registry | — | -| `url` | `string` | Git clone URL (HTTPS or SSH) | — | -| `syncInterval` | `number` | Auto-sync interval in seconds (0 = manual only) | `0` | -| `priority` | `number` | Conflict resolution priority — lower wins | `10` | -| `lastSyncedAt` | `string \| null` | ISO-8601 timestamp of last sync, null if never | `null` | +| Field | Type | Description | Default | +| -------------- | ---------------- | -------------------------------------------------------------- | ------- | +| `name` | `string` | Local name for the registry (`/^[a-zA-Z0-9_-]+$/`, 2-64 chars) | — | +| `url` | `string` | Git URL: `https://`, `ssh://`, `git@host:path` or `file:///` | — | +| `lastSyncedAt` | `string \| null` | ISO-8601 timestamp of last sync, null if never | `null` | You can edit this file directly or use `libscope registry add/remove`. @@ -315,15 +324,13 @@ To test access: `git ls-remote <registry-url>`. If that works, libscope will too ## Offline Behavior -Registries cache their index locally at `~/.libscope/registries/<name>/`. - -| Scenario | Behavior | -| ------------------------------------- | ------------------------------------------------------------------------------------------ | -| Registry unreachable, cache exists | Uses cached index with warning: "Registry '\<name\>' is unreachable. Using cached index from \<date\>." | -| Registry unreachable, no cache | Fails with: "Registry '\<name\>' has never been synced and is unreachable." | -| Cache stale, registry reachable | Auto-syncs before proceeding | +Each registry is cloned to `~/.libscope/registries/<name>/`. Searching, listing (`pack list --available`) and installing read only this local copy; they never use the network. The network is used only by `registry add`, `registry sync`, `registry publish` and `registry unpublish`. -Pack content downloads still require network access — only the index lookup can work offline. +| Scenario | Behavior | +| -------------------------------------------------------- | --------------------------------------------------------------------------- | +| `registry sync`, registry unreachable, local copy exists | Status `offline`: the local copy is kept and used | +| `registry sync`, registry unreachable, no local copy | Status `error` and exit code 1 | +| Search, list or install, registry never synced | The registry is skipped with a warning: run `libscope registry sync <name>` | --- @@ -332,7 +339,7 @@ Pack content downloads still require network access — only the index lookup ca Every pack version includes a `checksum.sha256` file containing the SHA-256 hex hash of the pack file. - **On publish**: libscope generates the checksum automatically and writes it alongside the pack file. -- **On install**: libscope verifies the checksum before extracting. A mismatch fails with: "Checksum verification failed — the pack file may have been tampered with or corrupted." +- **On install**: libscope verifies the checksum before extracting. A mismatch fails with a `ValidationError`: "Checksum verification failed for ...: expected ..., got .... The pack file may have been tampered with or corrupted." --- @@ -342,17 +349,11 @@ Pack versions follow [semver](https://semver.org/): - Versions must be valid semver strings (e.g. `1.0.0`, `2.3.1`) - `pack install <name>` installs the latest version -- `pack install <name>@1.0.0` or `--version 1.0.0` installs a specific version +- `pack install <name>@1.0.0` installs a specific version - Old versions are preserved in the registry — publishing a new version does not remove previous ones -- When publishing without `--version`, the patch version is auto-bumped from the latest +- When publishing without `--pack-version`, the patch version is auto-bumped from the latest - The `latestVersion` in `index.json` always points to the most recently published version -## Conflict Resolution - -When multiple registries contain a pack with the same name: - -- **Priority-based** (default): the registry with the lowest `priority` value wins -- **Explicit**: use `--registry <name>` to specify which registry to use -- **Interactive**: when running in a terminal without `--registry`, libscope prompts you to choose +## Packs in Several Registries -In non-interactive / CI mode, conflicts without `--registry` fail with an actionable error. +When more than one configured registry has a pack with the same name, `pack install <name>` fails with an error that names the registries. Choose one with `--registry <name>` (`registry` in the Node.js, REST and MCP interfaces). diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index a03bbe74..33c56f0f 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -3,142 +3,170 @@ Start the REST API server: ```bash -libscope serve --api --port 3378 +libscope serve api --port 3378 ``` -The OpenAPI 3.0 spec is available at `GET /openapi.json`. +Every route under `/api/v1` runs one LibScope operation: the same operation, with the same parameter names, defaults and validation, as the CLI, the MCP server and the SDK. The OpenAPI 3.1 document at `GET /openapi.json` is generated from the operations, so it always matches the server. -## Endpoints +## Conventions -### Search & Q&A +**Input.** Path parameters (`:documentId`) fill the input field of the same name. `GET` and `DELETE` routes read the other fields from the query string. Numbers and booleans are parsed (`limit=5`, `deleteDocuments=true`). Array fields take repeated or comma-separated values (`tags=a&tags=b` or `tags=a,b`). `POST`, `PATCH` and `PUT` routes read a JSON object body (maximum 1 MB). Unknown fields are ignored. -| Method | Endpoint | Description | -| ------ | ------------------------- | ------------------------------------- | -| `GET` | `/api/v1/search?q=...` | Semantic search | -| `POST` | `/api/v1/batch-search` | Run up to 20 search queries at once | -| `POST` | `/api/v1/ask` | RAG question-answering | +**Responses.** A success is `{ "data": <operation result>, "meta": { "took": <ms> } }` with status `200`. Lists are `{ "items": [...] }`, with `total`, `limit` and `offset` when the list is paged. Results that refer to a document carry `documentId`. Chunk results carry `chunkId`. -### Documents +**Background tasks.** Long-running operations (`add`, `sync`, `reindex`, `dedupe`, `install-pack`) start a task and answer `202` with `{ "data": { "taskId", "operation", "status" } }` and a `Location` header. Poll `GET /api/v1/tasks/:taskId`. When `status` is `completed`, `result` holds the operation result as a JSON string. When `status` is `failed`, `error` holds the message. The server keeps finished tasks in memory for one hour. -| Method | Endpoint | Description | -| -------- | ------------------------------------- | ------------------------------------- | -| `GET` | `/api/v1/documents` | List documents (with filters) | -| `POST` | `/api/v1/documents` | Index a new document | -| `GET` | `/api/v1/documents/:id` | Get a single document | -| `PATCH` | `/api/v1/documents/:id` | Update a document | -| `DELETE` | `/api/v1/documents/:id` | Delete a document | -| `POST` | `/api/v1/documents/url` | Index from a URL | -| `POST` | `/api/v1/documents/:id/tags` | Add tags to a document | -| `GET` | `/api/v1/documents/:id/suggest-tags` | Auto-suggest tags based on content | -| `GET` | `/api/v1/documents/:id/links` | List cross-reference links | -| `POST` | `/api/v1/documents/:id/links` | Create a cross-reference link | - -### Document Links - -| Method | Endpoint | Description | -| -------- | --------------------- | ---------------------- | -| `DELETE` | `/api/v1/links/:id` | Delete a link | - -### Topics & Tags - -| Method | Endpoint | Description | -| ------ | ----------------- | -------------------- | -| `GET` | `/api/v1/topics` | List all topics | -| `POST` | `/api/v1/topics` | Create a topic | -| `GET` | `/api/v1/tags` | List all tags | - -### Saved Searches - -| Method | Endpoint | Description | -| -------- | --------------------------- | ---------------------------- | -| `GET` | `/api/v1/searches` | List saved searches | -| `POST` | `/api/v1/searches` | Create a saved search | -| `POST` | `/api/v1/searches/:id/run` | Run a saved search | -| `DELETE` | `/api/v1/searches/:id` | Delete a saved search | - -### Bulk Operations - -| Method | Endpoint | Description | -| ------ | ----------------------- | ------------------------------ | -| `POST` | `/api/v1/bulk/delete` | Bulk delete documents | -| `POST` | `/api/v1/bulk/retag` | Bulk add/remove tags | -| `POST` | `/api/v1/bulk/move` | Bulk move documents to a topic | - -### Repository Indexing - -| Method | Endpoint | Description | -| ------ | ----------------------------------- | ------------------------------------------- | -| `POST` | `/api/v1/index/repos/:repoSlug` | Trigger a repo index job (async, 202) | -| `GET` | `/api/v1/index/jobs/:jobId` | Poll index job status | - -The `POST` endpoint is designed to be used as a **Bitbucket post-receive webhook**. It clones the repo, walks files, runs tree-sitter chunking for supported languages, and indexes everything into the main libscope database under `library: repoSlug`. The response is immediate (202) with a `jobId`. - -**Configure via `LIBSCOPE_REPOS_CONFIG`** — path to a JSON file: - -```json -{ - "repos": { - "my-service": { - "cloneUrl": "https://git.example.com/org/my-service.git", - "branch": "main", - "include": ["src/**/*.go", "**/*.cs"], - "exclude": ["vendor/**", "**/*_test.go"] - } - } -} -``` +**Errors.** An error is `{ "error": { "code": "...", "message": "..." } }`. -**Trigger a full reindex:** +| Status | When | +| ------ | ------------------------------------------------------------------------------------------------------ | +| `400` | Invalid input (`VALIDATION_ERROR`), invalid JSON (`INVALID_JSON`), or a malformed path | +| `401` | `LIBSCOPE_API_KEY` is set and the request has no `Authorization: Bearer <key>` header, or a wrong key | +| `403` | A browser page from another origin sent a write request (`FORBIDDEN_ORIGIN`) | +| `404` | Unknown route or method, or the resource does not exist (`DOCUMENT_NOT_FOUND`, `TOPIC_NOT_FOUND`, ...) | +| `413` | The body is larger than 1 MB (`PAYLOAD_TOO_LARGE`) | +| `429` | More than 120 requests per minute from one address (`RATE_LIMITED`) | +| `500` | Server error. A `CONFIG_ERROR` message tells you what to configure (for example, an LLM for `ask`) | +| `502` | A fetched URL failed (`FETCH_ERROR`) | -```bash -curl -X POST http://localhost:3378/api/v1/index/repos/my-service \ - -H "Authorization: Bearer $API_KEY" \ - -H "Content-Type: application/json" \ - -d '{}' -# → { "data": { "jobId": "abc-123", "status": "queued" } } -``` +**Authentication.** When the `LIBSCOPE_API_KEY` environment variable is set, every request must send `Authorization: Bearer <key>`. -**Incremental reindex** (specific files only, e.g. from a push webhook): +**CORS.** By default, only `http://localhost` and `http://localhost:3000` can call the API from a browser. To change the list, pass `corsOrigins` to `startApiServer`. The origin `"*"` lets every origin read (`GET`), but not write. A write request from an origin that is not in the list, and is not the server's own origin, gets `403`. -```bash -curl -X POST http://localhost:3378/api/v1/index/repos/my-service \ - -H "Authorization: Bearer $API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ "files": ["src/auth/handler.go", "src/models/user.go"] }' -``` +**Local files.** Over REST, `add` cannot read files or directories on the server, and `install-pack` cannot install a pack file. `backup`, `restore` and `create-pack` have no route. Use the CLI or the SDK for these. -**Poll job status:** +## Endpoints -```bash -curl http://localhost:3378/api/v1/index/jobs/abc-123 -# → { "data": { "jobId": "abc-123", "status": "completed", "stats": { ... } } } -``` +This list is generated from the routes (`npm run docs:gen`). The parameters of each route are in the OpenAPI document (`GET /openapi.json`). Routes with status `202` start a [background task](#conventions). -Job status values: `queued` → `running` → `completed` | `failed`. +<!-- generated:start rest:routes --> + +### Meta + +| Method | Path | Operation | Status | Description | +| ------ | ---------------- | --------- | ------ | --------------------------------------------------------------------- | +| `GET` | `/openapi.json` | — | `200` | This OpenAPI document | +| `GET` | `/api/v1/health` | — | `200` | Liveness check (use GET /api/v1/overview for counts and index health) | + +### Documents + +| Method | Path | Operation | Status | Description | +| -------- | ---------------------------------------- | ------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `POST` | `/api/v1/documents` | `add` | `202` (task) | Add content to the knowledge base: inline content, a URL (optionally crawled), a GitHub/GitLab repository, or (CLI/SDK only) a local file or directory | +| `GET` | `/api/v1/documents/:documentId` | `get-document` | `200` | Get a document with its tags, links and rating summary; long content can be paged | +| `GET` | `/api/v1/documents` | `list-documents` | `200` | List documents (newest first) with optional filters | +| `PATCH` | `/api/v1/documents/:documentId` | `update-document` | `200` | Update a document's title, content, metadata or tags (content changes are re-indexed) | +| `DELETE` | `/api/v1/documents/:documentId` | `delete-document` | `200` | Delete a document with its chunks, vectors, tags, links and ratings | +| `POST` | `/api/v1/documents/:documentId/ratings` | `rate-document` | `200` | Rate a document (1-5), optionally with feedback or a suggested correction | +| `GET` | `/api/v1/documents/:documentId/versions` | `document-history` | `200` | List saved versions of a document, newest first | +| `POST` | `/api/v1/documents/:documentId/rollback` | `rollback-document` | `200` | Restore a document to a saved version (the current state is saved first) | + +### Search and Q&A + +| Method | Path | Operation | Status | Description | +| ------ | ---------------- | --------- | ------ | --------------------------------------------------------------------------------------------------------- | +| `GET` | `/api/v1/search` | `search` | `200` | Search the knowledge base by meaning and keywords, or find content related to a document or chunk | +| `POST` | `/api/v1/ask` | `ask` | `200` | Answer a question from the knowledge base with an LLM, or (passthrough) return the context to answer from | + +### Links and graph + +| Method | Path | Operation | Status | Description | +| -------- | --------------------------------------------- | ------------------ | ------ | ----------------------------------------------------------------------------------------- | +| `POST` | `/api/v1/documents/:documentId/links` | `link-documents` | `200` | Create a typed link from one document to another | +| `DELETE` | `/api/v1/links/:linkId` | `unlink-documents` | `200` | Delete a link between documents | +| `GET` | `/api/v1/links` | `list-links` | `200` | List links of one document (outgoing and incoming) or all links | +| `GET` | `/api/v1/documents/:documentId/prerequisites` | `prerequisites` | `200` | List the documents to read before this one (following prerequisite links) | +| `GET` | `/api/v1/graph` | `graph` | `200` | Knowledge graph: documents, topics and tags as nodes; links, tags and similarity as edges | + +### Tags + +| Method | Path | Operation | Status | Description | +| -------- | ---------------------------------------------- | -------------- | ------ | --------------------------------------------------- | +| `POST` | `/api/v1/documents/:documentId/tags` | `add-tags` | `200` | Add tags to a document (tags are created as needed) | +| `DELETE` | `/api/v1/documents/:documentId/tags` | `remove-tags` | `200` | Remove tags from a document | +| `GET` | `/api/v1/tags` | `list-tags` | `200` | List all tags with their document counts | +| `GET` | `/api/v1/documents/:documentId/suggested-tags` | `suggest-tags` | `200` | Suggest tags for a document from its content | + +### Topics + +| Method | Path | Operation | Status | Description | +| -------- | ----------------------- | -------------- | ------ | --------------------------------------------- | +| `GET` | `/api/v1/topics` | `list-topics` | `200` | List topics with their document counts | +| `POST` | `/api/v1/topics` | `create-topic` | `200` | Create a topic | +| `DELETE` | `/api/v1/topics/:topic` | `delete-topic` | `200` | Delete a topic, optionally with its documents | + +### Saved searches + +| Method | Path | Operation | Status | Description | +| -------- | ------------------------------ | --------------------- | ------ | ------------------------------------------------ | +| `POST` | `/api/v1/searches` | `save-search` | `200` | Save a search query and its filters under a name | +| `GET` | `/api/v1/searches` | `list-saved-searches` | `200` | List saved searches | +| `POST` | `/api/v1/searches/:search/run` | `run-saved-search` | `200` | Run a saved search | +| `DELETE` | `/api/v1/searches/:search` | `delete-saved-search` | `200` | Delete a saved search | + +### Packs and registries + +| Method | Path | Operation | Status | Description | +| -------- | --------------------------- | ------------------- | ------------ | ----------------------------------------------------------------------------------- | +| `POST` | `/api/v1/packs` | `install-pack` | `202` (task) | Install a knowledge pack from a registry (name or name@version) | +| `DELETE` | `/api/v1/packs/:pack` | `remove-pack` | `200` | Remove an installed pack and its documents | +| `GET` | `/api/v1/packs` | `list-packs` | `200` | List installed packs, or the packs available in the configured registries | +| `GET` | `/api/v1/registries` | `list-registries` | `200` | List the configured pack registries | +| `GET` | `/api/v1/registries/search` | `search-registries` | `200` | Search the packs in the configured registries by name, description, tags and author | + +### Connectors + +| Method | Path | Operation | Status | Description | +| -------- | --------------------------- | ------------------ | ------------ | ------------------------------------------------------------------------------ | +| `GET` | `/api/v1/connections` | `list-connections` | `200` | List saved connector connections with their schedule and last sync | +| `POST` | `/api/v1/sync` | `sync` | `202` (task) | Sync one saved connection, or all of them, using the saved settings | +| `DELETE` | `/api/v1/connections/:name` | `disconnect` | `200` | Remove a connection's documents and its saved settings (including credentials) | + +### Admin and bulk + +| Method | Path | Operation | Status | Description | +| ------ | ----------------------------- | --------------- | ------------ | -------------------------------------------------------------------------------------------------- | +| `GET` | `/api/v1/overview` | `overview` | `200` | Knowledge base overview: counts, topics, installed packs, embedding model of the index, and health | +| `POST` | `/api/v1/admin/reindex` | `reindex` | `202` (task) | Re-embed chunks with the configured embedding model (rebuild after changing models) | +| `GET` | `/api/v1/admin/duplicates` | `dedupe` | `202` (task) | Find groups of duplicate or near-duplicate documents | +| `POST` | `/api/v1/admin/prune-expired` | `prune-expired` | `200` | Delete documents whose expiry time (expiresAt) has passed | +| `POST` | `/api/v1/bulk/delete` | `bulk-delete` | `200` | Delete every document matching the filters (at most 1000 per call) | +| `POST` | `/api/v1/bulk/retag` | `bulk-retag` | `200` | Add and/or remove tags on every document matching the filters | +| `POST` | `/api/v1/bulk/move` | `bulk-move` | `200` | Move every document matching the filters to another topic | + +### Analytics + +| Method | Path | Operation | Status | Description | +| ------ | ------------------------------- | ------------------ | ------ | -------------------------------------------------------------- | +| `GET` | `/api/v1/analytics/popular` | `popular` | `200` | Documents returned most often in search results | +| `GET` | `/api/v1/analytics/stale` | `stale` | `200` | Documents that no search returned in the given number of days | +| `GET` | `/api/v1/analytics/top-queries` | `top-queries` | `200` | Most frequent search queries | +| `GET` | `/api/v1/analytics/searches` | `search-analytics` | `200` | Search volume, top and zero-result queries, and knowledge gaps | ### Webhooks -| Method | Endpoint | Description | -| -------- | ------------------------------- | ----------------------------- | -| `GET` | `/api/v1/webhooks` | List webhooks | -| `POST` | `/api/v1/webhooks` | Create a webhook | -| `DELETE` | `/api/v1/webhooks/:id` | Delete a webhook | -| `POST` | `/api/v1/webhooks/:id/test` | Send a test ping | +| Method | Path | Operation | Status | Description | +| -------- | ---------------------------------- | ---------------- | ------ | ---------------------------------------------------------------------- | +| `POST` | `/api/v1/webhooks` | `create-webhook` | `200` | Register a URL to receive a signed POST for document and search events | +| `GET` | `/api/v1/webhooks` | `list-webhooks` | `200` | List registered webhooks (secrets are never shown) | +| `DELETE` | `/api/v1/webhooks/:webhookId` | `delete-webhook` | `200` | Delete a webhook | +| `POST` | `/api/v1/webhooks/:webhookId/test` | `test-webhook` | `200` | Send a test event to a webhook and report the HTTP status | + +### Tasks -### System +| Method | Path | Operation | Status | Description | +| ------ | ------------------------------ | ------------- | ------ | ------------------------------------------------------------ | +| `GET` | `/api/v1/tasks/:taskId` | `get-task` | `200` | Status, progress and result of a background task | +| `POST` | `/api/v1/tasks/:taskId/cancel` | `cancel-task` | `200` | Request cancellation of a pending or running background task | +| `GET` | `/api/v1/tasks` | `list-tasks` | `200` | List background tasks from the last hour, newest first | -| Method | Endpoint | Description | -| ------ | -------------------------------- | ---------------------------------- | -| `GET` | `/api/v1/health` | Health check with document count | -| `GET` | `/api/v1/stats` | Usage statistics | -| `GET` | `/api/v1/analytics/searches` | Search analytics and knowledge gaps| -| `GET` | `/api/v1/connectors/status` | Connector sync status and history | -| `GET` | `/api/v1/connectors/schedules` | Scheduled connector entries | -| `GET` | `/openapi.json` | OpenAPI 3.0 specification | +<!-- generated:end rest:routes --> + +Registries are read from the local copies under `~/.libscope/registries/`. Adding, removing, syncing and publishing to registries is done with the CLI or the SDK. ## Examples -### Index a document +### Add a document ```bash curl -X POST http://localhost:3378/api/v1/documents \ @@ -148,33 +176,37 @@ curl -X POST http://localhost:3378/api/v1/documents \ "content": "# Authentication\n\nUse OAuth2...", "tags": ["auth", "security"] }' +# -> 202 { "data": { "taskId": "6f1c...", "operation": "add", "status": "running" } } + +curl http://localhost:3378/api/v1/tasks/6f1c... +# -> { "data": { "status": "completed", "result": "{\"kind\":\"content\",\"documents\":[{\"documentId\":\"...\",...}],...}", ... } } ``` -### Search +### Add a web page, a crawled site or a repository ```bash -curl "http://localhost:3378/api/v1/search?q=authentication&limit=5" -``` +# One page +curl -X POST http://localhost:3378/api/v1/documents \ + -H "Content-Type: application/json" \ + -d '{ "url": "https://docs.example.com/guide", "library": "my-lib" }' -### Search with filters +# The page and the pages it links to +curl -X POST http://localhost:3378/api/v1/documents \ + -H "Content-Type: application/json" \ + -d '{ "url": "https://docs.example.com/", "spider": true, "maxPages": 50 }' -```bash -curl "http://localhost:3378/api/v1/search?q=deploy&library=my-lib&topic=backend&limit=10" +# A GitHub or GitLab repository (markdown and text files by default) +curl -X POST http://localhost:3378/api/v1/documents \ + -H "Content-Type: application/json" \ + -d '{ "source": "https://github.com/org/repo", "branch": "main", "paths": ["docs"] }' ``` -### Batch search - -Run multiple search queries concurrently (up to 20). Results are keyed by query string. +### Search ```bash -curl -X POST http://localhost:3378/api/v1/batch-search \ - -H "Content-Type: application/json" \ - -d '{ - "requests": [ - { "query": "authentication" }, - { "query": "deployment", "options": { "library": "my-lib", "limit": 5 } } - ] - }' +curl "http://localhost:3378/api/v1/search?query=authentication&limit=5" +curl "http://localhost:3378/api/v1/search?query=deploy&library=my-lib&topic=backend&tags=ops,prod" +curl "http://localhost:3378/api/v1/search?relatedTo=<documentId>" ``` ### Ask a question @@ -182,33 +214,26 @@ curl -X POST http://localhost:3378/api/v1/batch-search \ ```bash curl -X POST http://localhost:3378/api/v1/ask \ -H "Content-Type: application/json" \ - -d '{ - "question": "How does authentication work?", - "topic": "security" - }' + -d '{ "question": "How does authentication work?", "topic": "security" }' ``` -### Index from a URL +`ask` needs an LLM (see `llm.provider` in the [configuration reference](configuration.md)). Without one, `ask` answers `500` with the code `CONFIG_ERROR` and the setting to change. + +### Update a document ```bash -curl -X POST http://localhost:3378/api/v1/documents/url \ +curl -X PATCH http://localhost:3378/api/v1/documents/<documentId> \ -H "Content-Type: application/json" \ - -d '{ - "url": "https://docs.example.com/guide", - "library": "my-lib" - }' + -d '{ "title": "Auth Guide v2", "tags": ["auth"] }' ``` -### Update a document +### Tags ```bash -curl -X PATCH http://localhost:3378/api/v1/documents/<id> \ +curl -X POST http://localhost:3378/api/v1/documents/<documentId>/tags \ -H "Content-Type: application/json" \ - -d '{ - "title": "Updated Title", - "library": "my-lib", - "version": "2.0.0" - }' + -d '{ "tags": ["security"] }' +curl -X DELETE "http://localhost:3378/api/v1/documents/<documentId>/tags?tags=security" ``` ### Bulk retag @@ -216,55 +241,33 @@ curl -X PATCH http://localhost:3378/api/v1/documents/<id> \ ```bash curl -X POST http://localhost:3378/api/v1/bulk/retag \ -H "Content-Type: application/json" \ - -d '{ - "selector": {"library": "react"}, - "addTags": ["v18"], - "removeTags": ["v17"], - "dryRun": false - }' + -d '{ "library": "my-lib", "addTags": ["reviewed"], "dryRun": true }' ``` -### Create a webhook +### Link documents ```bash -curl -X POST http://localhost:3378/api/v1/webhooks \ +curl -X POST http://localhost:3378/api/v1/documents/<documentId>/links \ -H "Content-Type: application/json" \ - -d '{ - "url": "https://hooks.example.com/libscope", - "events": ["document.created", "document.updated"], - "secret": "my-hmac-secret" - }' + -d '{ "targetDocumentId": "<otherId>", "linkType": "prerequisite" }' +curl "http://localhost:3378/api/v1/links?documentId=<documentId>" ``` -Webhook payloads are signed with HMAC-SHA256 when a secret is provided. The signature is sent in the `X-LibScope-Signature` header. - -Supported events: `document.created`, `document.updated`, `document.deleted`. - -### Create a cross-reference link +### Saved searches ```bash -curl -X POST http://localhost:3378/api/v1/documents/<source-id>/links \ +curl -X POST http://localhost:3378/api/v1/searches \ -H "Content-Type: application/json" \ - -d '{ - "targetId": "<target-id>", - "linkType": "prerequisite", - "label": "Read this first" - }' + -d '{ "name": "auth-docs", "query": "authentication", "library": "my-lib" }' +curl -X POST http://localhost:3378/api/v1/searches/auth-docs/run ``` -Valid `linkType` values: `see_also`, `prerequisite`, `supersedes`, `related`. - -### Create a saved search +### Webhooks ```bash -curl -X POST http://localhost:3378/api/v1/searches \ +curl -X POST http://localhost:3378/api/v1/webhooks \ -H "Content-Type: application/json" \ - -d '{ - "name": "Auth Docs", - "query": "authentication best practices", - "filters": {"library": "my-lib"} - }' - -# Run it later -curl -X POST http://localhost:3378/api/v1/searches/<id>/run + -d '{ "url": "https://hooks.example.com/libscope", "events": ["document.created"] }' ``` + +See [Webhooks](../guide/webhooks.md) for the payload and the signature. diff --git a/package.json b/package.json index 55460ec9..24522f7c 100644 --- a/package.json +++ b/package.json @@ -8,12 +8,14 @@ "libscope": "./dist/cli/index.js" }, "exports": { - ".": "./dist/core/index.js", + ".": { + "types": "./dist/core/index.d.ts", + "default": "./dist/core/index.js" + }, "./mcp": "./dist/mcp/server.js", "./lite": { - "import": "./dist/lite/index.js", - "require": "./dist/lite/index.js", - "types": "./dist/lite/index.d.ts" + "types": "./dist/lite/index.d.ts", + "default": "./dist/lite/index.js" } }, "files": [ @@ -31,7 +33,9 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "prepare": "husky", - "serve": "node dist/mcp/server.js", + "serve": "node dist/mcp/main.js", + "docs:gen": "node scripts/gen-docs.mjs", + "docs:check": "node scripts/gen-docs.mjs --check", "docs:dev": "vitepress dev docs", "docs:build": "vitepress build docs", "docs:preview": "vitepress preview docs" diff --git a/scripts/gen-docs.mjs b/scripts/gen-docs.mjs new file mode 100644 index 00000000..10e34955 --- /dev/null +++ b/scripts/gen-docs.mjs @@ -0,0 +1,431 @@ +#!/usr/bin/env node +/** + * Fill the generated blocks of the reference docs from the built code (dist/). + * + * npm run build && npm run docs:gen rewrite the blocks + * npm run build && npm run docs:check exit 1 when a block is out of date + * + * A block is the text between `<!-- generated:start NAME -->` and `<!-- generated:end NAME -->`. + * Everything outside the blocks is hand-written and kept. Block names: + * + * cli:global docs/reference/cli.md global options of the `libscope` program + * cli:<cmd>[,<cmd>...] docs/reference/cli.md these top-level commands and their subcommands + * mcp:core, mcp:admin docs/reference/mcp-tools.md MCP tools with their parameters + * config:keys docs/reference/configuration.md every config key + * rest:routes docs/reference/rest-api.md every REST route + * + * The generator needs no network and no embedding model: it constructs the CLI program without + * running it, and builds the MCP server over an in-memory database with a stub provider. + */ +import { readFileSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { format, resolveConfig } from "prettier"; +import Database from "better-sqlite3"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const fromDist = (path) => import(pathToFileURL(join(ROOT, "dist", path)).href); + +const DOCS = { + cli: "docs/reference/cli.md", + mcp: "docs/reference/mcp-tools.md", + config: "docs/reference/configuration.md", + rest: "docs/reference/rest-api.md", +}; + +// --------------------------------------------------------------------------------------------- +// Markdown helpers + +/** A `|` escaped for a table cell, and the same between two alternatives. */ +const PIPE = String.raw`\|`; +const OR = ` ${PIPE} `; + +/** Text for a table cell or a paragraph: no HTML or Vue interpolation, no broken tables. */ +function text(value) { + return String(value ?? "") + .replaceAll(homedir(), "~") + .replaceAll("\n", " ") + .replaceAll("|", PIPE) + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("{{", "{ {"); +} + +/** Inline code; the content keeps `<` and `|` (escaped for tables). */ +function code(value) { + return `\`${String(value).replaceAll(homedir(), "~").replaceAll("|", PIPE)}\``; +} + +/** Text that ends with a period when `withPeriod` (when more text follows in the cell). */ +function sentence(value, withPeriod = true) { + const t = text(value).trim(); + if (t === "") return ""; + return withPeriod && !/[.!?]$/.test(t) ? `${t}.` : t; +} + +function table(headers, rows) { + const line = (cells) => `| ${cells.join(" | ")} |`; + return [line(headers), line(headers.map(() => "---")), ...rows.map(line)].join("\n"); +} + +// --------------------------------------------------------------------------------------------- +// CLI (commander program from dist/cli/index.js; importing it does not run it) + +function commandUsage(cmd, parents) { + const args = cmd.registeredArguments.map((a) => { + const name = `${a.name()}${a.variadic ? "..." : ""}`; + return a.required ? `<${name}>` : `[${name}]`; + }); + return [...parents, cmd.name(), ...args].join(" "); +} + +function optionDescription(opt) { + const extras = []; + if (opt.mandatory) extras.push("Required."); + if (opt.argChoices) extras.push(`One of: ${opt.argChoices.map(code).join(", ")}.`); + if (opt.defaultValue !== undefined && !opt.negate && !/\bdefault\b/i.test(opt.description)) { + extras.push( + `Default: ${code(opt.defaultValueDescription ?? JSON.stringify(opt.defaultValue))}.`, + ); + } + return [sentence(opt.description, extras.length > 0), ...extras].filter(Boolean).join(" "); +} + +function optionsTable(cmd) { + const options = cmd.options.filter((o) => !o.hidden); + if (options.length === 0) return ""; + return table( + ["Option", "Description"], + options.map((o) => [code(o.flags), optionDescription(o)]), + ); +} + +function renderLeafCommand(cmd, parents, heading) { + const parts = []; + const usage = `${code(commandUsage(cmd, parents))}`; + if (heading) parts.push(`${heading} ${usage}`, sentence(cmd.description())); + else parts.push(`Usage: ${usage}`, sentence(cmd.description())); + const options = optionsTable(cmd); + if (options) parts.push(options); + return parts.join("\n\n"); +} + +/** + * A top-level command. A command group renders each subcommand under an h3. A single command + * renders under an h3 when the block covers several commands, else without a heading (the + * hand-written h2 above the block names it). + */ +function renderCommand(cmd, alone) { + const visible = cmd.commands.filter((c) => !c._hidden); + if (visible.length === 0) return renderLeafCommand(cmd, ["libscope"], alone ? "" : "###"); + const parents = ["libscope", cmd.name()]; + return visible.map((sub) => renderLeafCommand(sub, parents, "###")).join("\n\n"); +} + +async function cliBlocks() { + const { program } = await fromDist("cli/index.js"); + const byName = new Map(program.commands.map((c) => [c.name(), c])); + const global = [ + "Usage: `libscope [options] <command>`", + optionsTable(program), + "`-h, --help` shows the help of the program or of any command.", + ].join("\n\n"); + return { + commands: program.commands.filter((c) => !c._hidden).map((c) => c.name()), + render(name) { + if (name === "cli:global") return global; + const names = name.slice("cli:".length).split(","); + return names + .map((n) => { + const cmd = byName.get(n); + if (!cmd) throw new Error(`${name}: the CLI has no command "${n}"`); + return renderCommand(cmd, names.length === 1); + }) + .join("\n\n"); + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// MCP tools (createMcpServer over an in-memory database; tools/list over an in-memory transport) + +function schemaType(s) { + if (!s || typeof s !== "object") return "any"; + if (s.enum) return s.enum.map((v) => code(v)).join(OR); + if (s.anyOf) return s.anyOf.map(schemaType).join(OR); + if (s.type === "array") return `${schemaType(s.items)}[]`; + if (s.type === "string" && s.format) return `string (${s.format})`; + return s.type ?? "any"; +} + +function annotationText(annotations = {}) { + if (annotations.readOnlyHint) return "read-only"; + return [ + annotations.destructiveHint ? "destructive" : "not destructive", + annotations.idempotentHint ? "idempotent" : null, + ] + .filter(Boolean) + .join(", "); +} + +function renderTool(tool) { + const parts = [`### ${tool.name}`, text(tool.description)]; + parts.push(`Annotations: ${annotationText(tool.annotations)}.`); + const props = tool.inputSchema?.properties ?? {}; + const required = new Set(tool.inputSchema?.required ?? []); + const rows = Object.entries(props).map(([key, s]) => { + // Many descriptions already state the default in words. + const statesDefault = /\bdefault\b/i.test(s.description ?? ""); + const def = + s.default !== undefined && !statesDefault + ? ` Default: ${code(JSON.stringify(s.default))}.` + : ""; + const description = [sentence(s.description, def !== ""), def.trim()].filter(Boolean).join(" "); + return [code(key), schemaType(s), required.has(key) ? "yes" : "", description]; + }); + parts.push( + rows.length ? table(["Parameter", "Type", "Required", "Description"], rows) : "No parameters.", + ); + return parts.join("\n\n"); +} + +async function listTools(ctx, createMcpServer, toolsets) { + const mcp = createMcpServer({ ctx, toolsets }); + const client = new Client({ name: "gen-docs", version: "0" }); + const [clientSide, serverSide] = InMemoryTransport.createLinkedPair(); + await Promise.all([mcp.server.connect(serverSide), client.connect(clientSide)]); + const { tools } = await client.listTools(); + await client.close(); + await mcp.close(); + return tools; +} + +async function mcpBlocks() { + const { createMcpServer } = await fromDist("mcp/server.js"); + const { createOperationContext } = await fromDist("core/operations/index.js"); + const { runMigrations } = await fromDist("db/schema.js"); + const { ConfigSchema } = await fromDist("config-schema.js"); + const { initLogger } = await fromDist("logger.js"); + initLogger("silent"); + + const db = new Database(":memory:"); + runMigrations(db); + const config = ConfigSchema.parse( + Object.fromEntries(Object.keys(ConfigSchema.shape).map((section) => [section, {}])), + ); + const provider = { + name: "docs", + dimensions: 4, + embed: async () => [0, 0, 0, 1], + embedBatch: async (texts) => texts.map(() => [0, 0, 0, 1]), + }; + // A stub LLM registers `ask` with its general description (the server registers it only + // when an LLM or passthrough is available). + const llm = { model: "docs", complete: async () => ({ text: "" }) }; + const ctx = createOperationContext({ db, provider, config, surface: "mcp", llm }); + + const core = await listTools(ctx, createMcpServer, []); + const all = await listTools(ctx, createMcpServer, ["admin"]); + db.close(); + const coreNames = new Set(core.map((t) => t.name)); + const admin = all.filter((t) => !coreNames.has(t.name)); + return { + render(name) { + if (name === "mcp:core") return core.map(renderTool).join("\n\n"); + if (name === "mcp:admin") return admin.map(renderTool).join("\n\n"); + throw new Error(`unknown block ${name}`); + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// Config keys (getConfigKeyTable from dist/config-schema.js) + +/** "a | b" -> `a` \| `b`; "list of a | b" -> list of `a` \| `b`; "string" stays. */ +function configType(type) { + const list = type.startsWith("list of "); + const values = list ? type.slice("list of ".length) : type; + if (!list && !values.includes(" | ")) return values; + const choices = values.split(" | ").map(code).join(OR); + return list ? `list of ${choices}` : choices; +} + +/** A single value in code; in a sentence, only the URLs (a bare URL would become a link). */ +function defaultCell(value) { + if (value === "") return "—"; + if (!/\s/.test(value)) return code(value); + return text(value).replaceAll(/https?:\/\/[^\s,;]+/g, (url) => `\`${url}\``); +} + +async function configBlocks() { + const { getConfigKeyTable } = await fromDist("config-schema.js"); + const rows = getConfigKeyTable().map((row) => [ + code(row.key), + configType(row.type), + defaultCell(row.default), + row.env.map(code).join(", "), + row.secret ? "yes" : "", + sentence(row.description, false), + ]); + const keys = table( + ["Key", "Type", "Default", "Environment variables", "Secret", "Description"], + rows, + ); + return { + render(name) { + if (name === "config:keys") return keys; + throw new Error(`unknown block ${name}`); + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// REST routes (API_ROUTES from dist/api/routes.js) + +const ROUTE_GROUPS = [ + ["meta", "Meta"], + ["documents", "Documents"], + ["search", "Search and Q&A"], + ["links", "Links and graph"], + ["tags", "Tags"], + ["topics", "Topics"], + ["searches", "Saved searches"], + ["packs", "Packs and registries"], + ["connectors", "Connectors"], + ["admin", "Admin and bulk"], + ["analytics", "Analytics"], + ["webhooks", "Webhooks"], + ["tasks", "Tasks"], +]; + +async function restBlocks() { + const { API_ROUTES } = await fromDist("api/routes.js"); + const groups = new Map(ROUTE_GROUPS.map(([key]) => [key, []])); + for (const route of API_ROUTES) { + const group = route.operation?.group ?? "meta"; + if (!groups.has(group)) + throw new Error(`REST route group "${group}" has no heading in gen-docs`); + groups.get(group).push(route); + } + const sections = ROUTE_GROUPS.filter(([key]) => groups.get(key).length > 0).map( + ([key, title]) => { + const rows = groups + .get(key) + .map((r) => [ + code(r.method), + code(r.path), + r.operation ? code(r.operation.name) : "—", + r.operation?.annotations?.longRunning ? "`202` (task)" : "`200`", + sentence(r.summary, false), + ]); + return `### ${title}\n\n${table(["Method", "Path", "Operation", "Status", "Description"], rows)}`; + }, + ); + const routes = sections.join("\n\n"); + return { + render(name) { + if (name === "rest:routes") return routes; + throw new Error(`unknown block ${name}`); + }, + }; +} + +// --------------------------------------------------------------------------------------------- +// Blocks in files + +const BLOCK = /<!-- generated:start ([\w:,-]+) -->\n([\s\S]*?)<!-- generated:end \1 -->/g; + +async function formatBlock(content, file) { + const options = (await resolveConfig(file)) ?? {}; + return format(content, { ...options, parser: "markdown", filepath: file }); +} + +const REQUIRED_BLOCKS = { + cli: ["cli:global"], + mcp: ["mcp:core", "mcp:admin"], + config: ["config:keys"], + rest: ["rest:routes"], +}; + +/** The generated text of a block, with blank lines around it as prettier writes them. */ +async function blockBody(generator, name, file) { + return `\n${await formatBlock(generator.render(name), file)}\n`; +} + +/** Each top-level command needs a `cli:<cmd>` block. */ +function uncoveredCommands(relPath, seen, commands) { + const covered = new Set( + [...seen].filter((n) => n !== "cli:global").flatMap((n) => n.slice(4).split(",")), + ); + return commands + .filter((command) => !covered.has(command)) + .map((command) => `${relPath}: no block documents "libscope ${command}"`); +} + +/** Regenerate the blocks of one doc file: its original and updated text, and its problems. */ +async function updateFile(kind, relPath, generator, check) { + const file = join(ROOT, relPath); + const original = readFileSync(file, "utf8"); + const matches = [...original.matchAll(BLOCK)]; + const bodies = await Promise.all( + matches.map(async ([, name]) => + name.split(":")[0] === kind ? blockBody(generator, name, file) : null, + ), + ); + const problems = []; + const seen = new Set(); + let updated = original; + matches.forEach(([whole, name, current], i) => { + const body = bodies[i]; + if (body === null) { + problems.push(`${relPath}: block "${name}" does not belong in this file`); + return; + } + seen.add(name); + if (current === body) return; + if (check) problems.push(`${relPath}: block "${name}" is out of date`); + const block = `<!-- generated:start ${name} -->\n${body}<!-- generated:end ${name} -->`; + updated = updated.replace(whole, () => block); + }); + for (const name of REQUIRED_BLOCKS[kind]) { + if (!seen.has(name)) problems.push(`${relPath}: missing block "${name}"`); + } + if (kind === "cli") problems.push(...uncoveredCommands(relPath, seen, generator.commands)); + return { file, original, updated, problems }; +} + +async function main() { + const check = process.argv.includes("--check"); + const generators = { + cli: await cliBlocks(), + mcp: await mcpBlocks(), + config: await configBlocks(), + rest: await restBlocks(), + }; + const results = await Promise.all( + Object.entries(DOCS).map(([kind, relPath]) => + updateFile(kind, relPath, generators[kind], check), + ), + ); + if (!check) { + for (const { file, original, updated } of results) { + if (updated === original) continue; + writeFileSync(file, updated); + console.log(`updated ${relative(ROOT, file)}`); + } + } + + const problems = results.flatMap((r) => r.problems); + if (problems.length > 0) { + for (const p of problems) console.error(`✗ ${p}`); + if (check) console.error("Run `npm run build && npm run docs:gen` and commit the result."); + process.exitCode = 1; + } else if (check) { + console.log("Generated docs are up to date."); + } +} + +await main(); diff --git a/sdk/go/README.md b/sdk/go/README.md index 88e4c664..45913877 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -1,6 +1,6 @@ # libscope Go SDK -A lightweight, idiomatic Go client for the [libscope](https://github.com/RobertLD/libscope) REST API. Zero external dependencies — built entirely on the Go standard library. +A small Go client for the [libscope](https://github.com/RobertLD/libscope) REST API (`/api/v1`). It uses only the Go standard library. ## Installation @@ -8,6 +8,8 @@ A lightweight, idiomatic Go client for the [libscope](https://github.com/RobertL go get github.com/RobertLD/libscope/sdk/go ``` +Start a server with `libscope serve api` (default `http://localhost:3378`). + ## Quick Start ```go @@ -17,46 +19,36 @@ import ( "context" "fmt" "log" + "time" libscope "github.com/RobertLD/libscope/sdk/go" ) func main() { + ctx := context.Background() client := libscope.NewClient() - // Add a document from a URL - doc, err := client.AddDocument(context.Background(), "https://go.dev/doc/") + // Adding runs as a background task on the server. WaitForTask polls it. + task, err := client.AddURL(ctx, "https://go.dev/doc/", libscope.WithTags("go")) if err != nil { log.Fatal(err) } - fmt.Printf("Indexed: %s (%s)\n", doc.Title, doc.ID) - - // Add a text document - doc, err = client.AddText(context.Background(), "Go Concurrency", "Goroutines are lightweight threads...") + task, err = client.WaitForTask(ctx, task.ID, time.Second) if err != nil { log.Fatal(err) } - fmt.Printf("Added: %s\n", doc.ID) - - // Search - results, err := client.Search(context.Background(), "goroutines", - libscope.WithLimit(5), - libscope.WithMinScore(0.5), - ) - if err != nil { + var added libscope.IngestResult + if err := task.DecodeResult(&added); err != nil { log.Fatal(err) } - for _, hit := range results.Results { - fmt.Printf(" %s: %.2f\n", hit.Document.Title, hit.Score) - } + fmt.Printf("Indexed: %s (%s)\n", added.Documents[0].Title, added.Documents[0].DocumentID) - // List all topics - topics, err := client.ListTopics(context.Background()) + page, err := client.Search(ctx, "goroutines", libscope.WithLimit(5)) if err != nil { log.Fatal(err) } - for _, t := range topics { - fmt.Printf(" Topic: %s\n", t.Name) + for _, hit := range page.Items { + fmt.Printf(" %s: %.2f\n", hit.Title, hit.Score) } } ``` @@ -64,76 +56,56 @@ func main() { ## Configuration ```go -// Custom base URL client := libscope.NewClient( libscope.WithBaseURL("http://my-server:3378"), -) - -// Custom timeout -client := libscope.NewClient( - libscope.WithTimeout(10 * time.Second), -) - -// Custom HTTP client -client := libscope.NewClient( - libscope.WithHTTPClient(&http.Client{ - Transport: myTransport, - }), + libscope.WithTimeout(10*time.Second), + libscope.WithAPIKey(os.Getenv("LIBSCOPE_API_KEY")), // "Authorization: Bearer <key>" + libscope.WithHTTPClient(&http.Client{Transport: myTransport}), ) ``` -## API Reference +## Background tasks -### Documents +`AddText`, `AddURL`, `Sync` and `SyncAll` return a `*Task` at once (the server answers `202`). `WaitForTask(ctx, taskID, interval)` polls `GET /api/v1/tasks/:taskId` until the task finishes. It returns `*TaskError` when the task failed or was cancelled. Bound the wait with `context.WithTimeout`. `Task.DecodeResult` decodes the operation result (`*IngestResult` for `AddText` and `AddURL`). `GetTask` and `CancelTask` are also available. -| Method | Description | -| --------------------------------------- | ------------------------------------ | -| `AddDocument(ctx, url, opts...)` | Index a document from a URL | -| `AddText(ctx, title, content, opts...)` | Index raw text content | -| `GetDocument(ctx, id)` | Get a document by ID | -| `ListDocuments(ctx, opts...)` | List documents with optional filters | -| `DeleteDocument(ctx, id)` | Delete a document | - -### Search - -| Method | Description | -| ----------------------------- | -------------------------------- | -| `Search(ctx, query, opts...)` | Semantic search across documents | - -Search options: `WithLimit(n)`, `WithTopic(t)`, `WithTags(tags...)`, `WithMinScore(s)` - -### Topics - -| Method | Description | -| --------------------------------- | ------------------ | -| `ListTopics(ctx)` | List all topics | -| `CreateTopic(ctx, name, opts...)` | Create a new topic | - -### Tags - -| Method | Description | -| ------------------------------------- | ---------------------- | -| `ListTags(ctx)` | List all tags | -| `AddTagsToDocument(ctx, docID, tags)` | Add tags to a document | - -### Analytics +## API Reference -| Method | Description | -| --------------- | ----------------------- | -| `GetStats(ctx)` | Get instance statistics | -| `Health(ctx)` | Health check | +Optional fields are `Param` values, for example `WithTopic("go")`. The same `Param` works on every method that accepts the field. + +| Method | Description | Params | +| ----------------------------------------- | -------------------------------------------------------------------------------------- | ------------------------------------------------------- | +| `Search(ctx, query, params...)` | Search by meaning and keywords (`*Page[SearchHit]`) | `WithLimit`, `WithOffset`, `WithMinRating`, filters | +| `Ask(ctx, question, params...)` | Answer a question from the knowledge base (the server needs an LLM) | `WithTopK`, `WithMinRating`, filters | +| `AddText(ctx, title, content, params...)` | Start adding a document from text (`*Task`) | `WithURL`, filters | +| `AddURL(ctx, url, params...)` | Start adding a web page, a crawled site or a public GitHub/GitLab repository (`*Task`) | `WithSpider`, `WithMaxPages`, filters | +| `GetDocument(ctx, id)` | Document with content, tags, links and ratings (`*DocumentView`) | | +| `ListDocuments(ctx, params...)` | List documents (`*Page[Document]`) | `WithLimit`, `WithOffset`, filters | +| `DeleteDocument(ctx, id)` | Delete a document | | +| `ListTopics(ctx)` | Topics with document counts | | +| `CreateTopic(ctx, name, params...)` | Create a topic | `WithParent`, `WithDescription` | +| `ListTags(ctx)` | Tags with document counts | | +| `AddTags(ctx, id, tags)` | Add tags; returns the document's tags | | +| `RemoveTags(ctx, id, tags)` | Remove tags; returns the document's tags | | +| `GetGraph(ctx, params...)` | Knowledge graph | `WithTopic`, `WithTag`, `WithThreshold`, `WithMaxNodes` | +| `Sync(ctx, name)` / `SyncAll(ctx)` | Start syncing saved connector connections (`*Task`) | | +| `GetTask` / `CancelTask` / `WaitForTask` | Background tasks | | +| `Overview(ctx)` | Counts, topics, installed packs, index and health | | +| `Health(ctx)` | Liveness check | | + +Filters: `WithTopic`, `WithLibrary`, `WithVersion`, `WithSourceType`, `WithTags`. On `AddText` and `AddURL` they set the new document's fields. ## Error Handling -All methods return `(*T, error)`. API errors are returned as `*libscope.Error`: +API errors are returned as `*libscope.Error`. `errors.Is` matches by HTTP status (`ErrNotFound`, `ErrBadRequest`, `ErrServerError`): ```go -doc, err := client.GetDocument(ctx, "nonexistent") -if err != nil { - var apiErr *libscope.Error - if errors.As(err, &apiErr) { - fmt.Printf("API error %d: %s\n", apiErr.StatusCode, apiErr.Message) - } +_, err := client.GetDocument(ctx, "nonexistent") +if errors.Is(err, libscope.ErrNotFound) { + // ... +} +var apiErr *libscope.Error +if errors.As(err, &apiErr) { + fmt.Printf("API error %d %s: %s\n", apiErr.StatusCode, apiErr.Code, apiErr.Message) } ``` @@ -144,4 +116,4 @@ cd sdk/go go test ./... -v -count=1 ``` -All tests use `httptest.NewServer` — no running libscope instance required. +The tests use `httptest.NewServer`. They do not need a running libscope server. diff --git a/sdk/go/analytics.go b/sdk/go/analytics.go deleted file mode 100644 index 6b5d79c0..00000000 --- a/sdk/go/analytics.go +++ /dev/null @@ -1,19 +0,0 @@ -package libscope - -import ( - "context" - "net/http" -) - -// GetStats retrieves analytics/statistics about the libscope instance. -func (c *Client) GetStats(ctx context.Context) (*Stats, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/stats", nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[Stats](resp) - if err != nil { - return nil, err - } - return &result, nil -} diff --git a/sdk/go/client.go b/sdk/go/client.go index e2819d68..b9192928 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -1,11 +1,13 @@ package libscope import ( + "bytes" "context" "encoding/json" "fmt" "io" "net/http" + "net/url" "strings" "time" ) @@ -13,21 +15,23 @@ import ( const ( defaultBaseURL = "http://localhost:3378" defaultTimeout = 30 * time.Second + apiPrefix = "/api/v1" ) -// Client is the libscope API client. +// Client is the libscope REST API client. type Client struct { baseURL string + apiKey string httpClient *http.Client } // Option configures a Client. type Option func(*Client) -// WithBaseURL sets the base URL for the API. -func WithBaseURL(url string) Option { +// WithBaseURL sets the server URL (default http://localhost:3378). +func WithBaseURL(u string) Option { return func(c *Client) { - c.baseURL = strings.TrimRight(url, "/") + c.baseURL = strings.TrimRight(u, "/") } } @@ -45,7 +49,14 @@ func WithHTTPClient(hc *http.Client) Option { } } -// NewClient creates a new libscope API client with the given options. +// WithAPIKey sends "Authorization: Bearer <key>" (needed when the server sets LIBSCOPE_API_KEY). +func WithAPIKey(key string) Option { + return func(c *Client) { + c.apiKey = key + } +} + +// NewClient creates a libscope API client. func NewClient(opts ...Option) *Client { c := &Client{ baseURL: defaultBaseURL, @@ -57,65 +68,84 @@ func NewClient(opts ...Option) *Client { return c } -// do executes an HTTP request and returns the response body. -func (c *Client) do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { - url := c.baseURL + path - req, err := http.NewRequestWithContext(ctx, method, url, body) +// call sends a request to /api/v1+path and decodes the "data" envelope into out (if non-nil). +// Non-2xx responses are returned as *Error. +func (c *Client) call(ctx context.Context, method, path string, query url.Values, body, out any) error { + target := c.baseURL + apiPrefix + path + if len(query) > 0 { + target += "?" + query.Encode() + } + var reader io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("libscope: encoding request: %w", err) + } + reader = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, target, reader) if err != nil { - return nil, fmt.Errorf("libscope: creating request: %w", err) + return fmt.Errorf("libscope: creating request: %w", err) } if body != nil { req.Header.Set("Content-Type", "application/json") } + if c.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+c.apiKey) + } resp, err := c.httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("libscope: executing request: %w", err) + return fmt.Errorf("libscope: executing request: %w", err) } - return resp, nil -} - -// decodeResponse reads the response body and decodes the API envelope. -// On non-2xx status, it returns an *Error. -func decodeResponse[T any](resp *http.Response) (T, error) { - var zero T defer resp.Body.Close() - data, err := io.ReadAll(resp.Body) if err != nil { - return zero, fmt.Errorf("libscope: reading response: %w", err) + return fmt.Errorf("libscope: reading response: %w", err) } - if resp.StatusCode >= 400 { - var errResp apiErrorResponse - if json.Unmarshal(data, &errResp) == nil && errResp.Error.Message != "" { - return zero, &Error{ - StatusCode: resp.StatusCode, - Code: errResp.Error.Code, - Message: errResp.Error.Message, - } - } - return zero, &Error{ - StatusCode: resp.StatusCode, - Message: string(data), - } + return newError(resp.StatusCode, data) } - - var envelope apiResponse[T] + if out == nil { + return nil + } + envelope := struct { + Data any `json:"data"` + }{Data: out} if err := json.Unmarshal(data, &envelope); err != nil { - return zero, fmt.Errorf("libscope: decoding response: %w", err) + return fmt.Errorf("libscope: decoding response: %w", err) } - return envelope.Data, nil + return nil } -// Health checks the health of the libscope server. -func (c *Client) Health(ctx context.Context) (*HealthStatus, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/health", nil) - if err != nil { +// get decodes a GET response into a new T. +func get[T any](ctx context.Context, c *Client, path string, query url.Values) (*T, error) { + var out T + if err := c.call(ctx, http.MethodGet, path, query, nil, &out); err != nil { return nil, err } - result, err := decodeResponse[HealthStatus](resp) - if err != nil { + return &out, nil +} + +// send decodes a response with a JSON body into a new T. +func send[T any](ctx context.Context, c *Client, method, path string, body any) (*T, error) { + var out T + if err := c.call(ctx, method, path, nil, body, &out); err != nil { return nil, err } - return &result, nil + return &out, nil +} + +// segment escapes one path segment (an ID or a name). +func segment(s string) string { + return "/" + url.PathEscape(s) +} + +// Health checks that the server is up. +func (c *Client) Health(ctx context.Context) (*HealthStatus, error) { + return get[HealthStatus](ctx, c, "/health", nil) +} + +// Overview returns counts, topics, installed packs, the embedding index and health. +func (c *Client) Overview(ctx context.Context) (*Overview, error) { + return get[Overview](ctx, c, "/overview", nil) } diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 47fdd0c5..4aa46ef4 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -2,105 +2,261 @@ package libscope import ( "context" + "encoding/json" + "errors" + "io" "net/http" "net/http/httptest" + "reflect" "testing" "time" ) -func TestNewClientDefaults(t *testing.T) { - c := NewClient() - if c.baseURL != defaultBaseURL { - t.Errorf("expected base URL %q, got %q", defaultBaseURL, c.baseURL) - } - if c.httpClient.Timeout != defaultTimeout { - t.Errorf("expected timeout %v, got %v", defaultTimeout, c.httpClient.Timeout) - } -} - -func TestNewClientWithOptions(t *testing.T) { - c := NewClient( - WithBaseURL("http://example.com/"), - WithTimeout(5*time.Second), - ) - if c.baseURL != "http://example.com" { - t.Errorf("expected base URL %q, got %q", "http://example.com", c.baseURL) - } - if c.httpClient.Timeout != 5*time.Second { - t.Errorf("expected timeout 5s, got %v", c.httpClient.Timeout) - } +// request is what the fake server saw. +type request struct { + Method string + Path string + Query map[string][]string + Body map[string]any + Header http.Header } -func TestWithHTTPClient(t *testing.T) { - custom := &http.Client{Timeout: 10 * time.Second} - c := NewClient(WithHTTPClient(custom)) - if c.httpClient != custom { - t.Error("expected custom HTTP client") - } +// fakeServer answers each request with the next reply ({"data": ...} with status 200 unless +// set) and records the requests. +type reply struct { + status int + data any + raw string } -func TestHealth(t *testing.T) { +func fakeServer(t *testing.T, replies ...reply) (*Client, *[]request) { + t.Helper() + var seen []request srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/v1/health" { - t.Errorf("unexpected path: %s", r.URL.Path) + req := request{Method: r.Method, Path: r.URL.EscapedPath(), Query: r.URL.Query(), Header: r.Header} + if data, _ := io.ReadAll(r.Body); len(data) > 0 { + if err := json.Unmarshal(data, &req.Body); err != nil { + t.Errorf("request body is not a JSON object: %s", data) + } + } + seen = append(seen, req) + if len(replies) == 0 { + t.Errorf("unexpected request %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusInternalServerError) + return + } + rep := replies[0] + if len(replies) > 1 { + replies = replies[1:] } w.Header().Set("Content-Type", "application/json") - w.WriteHeader(200) - w.Write([]byte(`{"data":{"status":"ok","docCount":42,"dbSize":1024},"meta":{"took":5}}`)) + if rep.status != 0 { + w.WriteHeader(rep.status) + } + if rep.raw != "" { + _, _ = w.Write([]byte(rep.raw)) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": rep.data, "meta": map[string]any{"took": 1}}) })) - defer srv.Close() + t.Cleanup(srv.Close) + return NewClient(WithBaseURL(srv.URL + "/")), &seen +} + +func check(t *testing.T, label string, got, want any) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: got %#v, want %#v", label, got, want) + } +} - c := NewClient(WithBaseURL(srv.URL)) - health, err := c.Health(context.Background()) +func must(t *testing.T, err error) { + t.Helper() if err != nil { t.Fatalf("unexpected error: %v", err) } - if health.Status != "ok" { - t.Errorf("expected status ok, got %q", health.Status) - } - if health.DocCount != 42 { - t.Errorf("expected 42 docs, got %d", health.DocCount) - } } -func TestContextCancellation(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(2 * time.Second) - w.WriteHeader(200) - })) - defer srv.Close() +var ctx = context.Background() - c := NewClient(WithBaseURL(srv.URL)) - ctx, cancel := context.WithCancel(context.Background()) - cancel() // cancel immediately +func TestNewClientOptions(t *testing.T) { + c := NewClient() + check(t, "base URL", c.baseURL, defaultBaseURL) + check(t, "timeout", c.httpClient.Timeout, defaultTimeout) + custom := &http.Client{} + c = NewClient(WithBaseURL("http://example.com/"), WithHTTPClient(custom), WithTimeout(5*time.Second)) + check(t, "base URL", c.baseURL, "http://example.com") + check(t, "http client", c.httpClient == custom, true) + check(t, "timeout", custom.Timeout, 5*time.Second) +} - _, err := c.Health(ctx) - if err == nil { - t.Fatal("expected error from cancelled context") - } +func TestHealthAndAPIKey(t *testing.T) { + c, seen := fakeServer(t, reply{data: map[string]any{"status": "ok"}}) + WithAPIKey("secret")(c) + health, err := c.Health(ctx) + must(t, err) + check(t, "status", health.Status, "ok") + check(t, "path", (*seen)[0].Path, "/api/v1/health") + check(t, "auth", (*seen)[0].Header.Get("Authorization"), "Bearer secret") } -func TestErrorResponse(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(500) - w.Write([]byte(`{"error":{"code":"INTERNAL_ERROR","message":"something broke"}}`)) - })) - defer srv.Close() +func TestSearchSendsOperationNames(t *testing.T) { + c, seen := fakeServer(t, reply{data: map[string]any{ + "items": []any{map[string]any{"documentId": "d1", "chunkId": "c1", "title": "Go", "score": 0.9}}, + "total": 1, "limit": 5, "offset": 0, + }}) + page, err := c.Search(ctx, "goroutines", WithLimit(5), WithSourceType("library"), WithTags("a", "b"), WithMinRating(3)) + must(t, err) + q := (*seen)[0].Query + check(t, "query", q["query"], []string{"goroutines"}) + check(t, "limit", q["limit"], []string{"5"}) + check(t, "sourceType", q["sourceType"], []string{"library"}) + check(t, "tags", q["tags"], []string{"a", "b"}) + check(t, "minRating", q["minRating"], []string{"3"}) + check(t, "total", page.Total, 1) + check(t, "hit", page.Items[0].ChunkID, "c1") +} + +func TestAddTextWaitAndDecode(t *testing.T) { + result := `{"kind":"content","documents":[{"documentId":"d1","title":"Go","chunkCount":2}],"errors":[],"skipped":[]}` + c, seen := fakeServer(t, + reply{status: 202, data: map[string]any{"taskId": "t1", "operation": "add", "status": "running"}}, + reply{data: map[string]any{"id": "t1", "operation": "add", "status": "running"}}, + reply{data: map[string]any{"id": "t1", "operation": "add", "status": "completed", "result": result}}, + ) + task, err := c.AddText(ctx, "Go", "content", WithTopic("lang"), WithTags("go")) + must(t, err) + check(t, "task", *task, Task{ID: "t1", Operation: "add", Status: "running"}) + check(t, "body", (*seen)[0].Body, map[string]any{ + "title": "Go", "content": "content", "topic": "lang", "tags": []any{"go"}, + }) + done, err := c.WaitForTask(ctx, task.ID, time.Millisecond) + must(t, err) + check(t, "poll path", (*seen)[2].Path, "/api/v1/tasks/t1") + var added IngestResult + must(t, done.DecodeResult(&added)) + check(t, "document", added.Documents[0], IngestedDocument{DocumentID: "d1", Title: "Go", ChunkCount: 2}) +} + +func TestAddURLWithSpider(t *testing.T) { + c, seen := fakeServer(t, reply{status: 202, data: map[string]any{"taskId": "t", "status": "running"}}) + _, err := c.AddURL(ctx, "https://go.dev/doc/", WithSpider(), WithMaxPages(10)) + must(t, err) + check(t, "body", (*seen)[0].Body, map[string]any{"url": "https://go.dev/doc/", "spider": true, "maxPages": float64(10)}) +} - c := NewClient(WithBaseURL(srv.URL)) - _, err := c.Health(context.Background()) - if err == nil { - t.Fatal("expected error") +func TestWaitForTaskFailures(t *testing.T) { + c, _ := fakeServer(t, reply{data: map[string]any{"id": "t1", "status": "failed", "error": "boom"}}) + _, err := c.WaitForTask(ctx, "t1", time.Millisecond) + var taskErr *TaskError + if !errors.As(err, &taskErr) || taskErr.Task.Error != "boom" { + t.Fatalf("expected TaskError with boom, got %v", err) } - apiErr, ok := err.(*Error) - if !ok { - t.Fatalf("expected *Error, got %T", err) + + c, _ = fakeServer(t, reply{data: map[string]any{"id": "t1", "status": "running"}}) + short, cancel := context.WithTimeout(ctx, 20*time.Millisecond) + defer cancel() + _, err = c.WaitForTask(short, "t1", time.Millisecond) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected deadline exceeded, got %v", err) + } +} + +func TestDocuments(t *testing.T) { + doc := map[string]any{"documentId": "d/1", "title": "Go", "sourceType": "manual"} + c, seen := fakeServer(t, + reply{data: map[string]any{"document": doc, "content": "body", "tags": []string{"go"}, "nextOffset": nil}}, + reply{data: map[string]any{"items": []any{doc}, "total": 1, "limit": 10, "offset": 0}}, + reply{data: map[string]any{"documentId": "d/1", "deleted": true}}, + ) + view, err := c.GetDocument(ctx, "d/1") + must(t, err) + check(t, "escaped path", (*seen)[0].Path, "/api/v1/documents/d%2F1") + check(t, "view", []any{view.Document.Title, view.Content, view.Tags, view.NextOffset == nil}, []any{"Go", "body", []string{"go"}, true}) + page, err := c.ListDocuments(ctx, WithLimit(10), WithLibrary("react")) + must(t, err) + check(t, "library", (*seen)[1].Query["library"], []string{"react"}) + check(t, "list", page.Items[0].DocumentID, "d/1") + must(t, c.DeleteDocument(ctx, "d/1")) + check(t, "delete", (*seen)[2].Method, http.MethodDelete) +} + +func TestTopicsTagsGraph(t *testing.T) { + c, seen := fakeServer(t, + reply{data: map[string]any{"items": []any{map[string]any{"id": "go", "name": "Go", "documentCount": 2}}}}, + reply{data: map[string]any{"id": "web", "name": "Web", "parentId": "go"}}, + reply{data: map[string]any{"items": []any{map[string]any{"id": "1", "name": "a", "documentCount": 1}}}}, + reply{data: map[string]any{"documentId": "d1", "tags": []string{"a", "b"}}}, + reply{data: map[string]any{"documentId": "d1", "removed": []string{"b"}, "tags": []string{"a"}}}, + reply{data: map[string]any{ + "nodes": []any{map[string]any{"id": "d1", "label": "Doc", "type": "document"}}, + "edges": []any{map[string]any{"source": "d1", "target": "t", "type": "has_tag", "weight": 1}}, + }}, + ) + topics, err := c.ListTopics(ctx) + must(t, err) + check(t, "topic count", topics[0].DocumentCount, 2) + topic, err := c.CreateTopic(ctx, "Web", WithParent("go")) + must(t, err) + check(t, "create body", (*seen)[1].Body, map[string]any{"name": "Web", "parent": "go"}) + check(t, "topic", topic.ParentID, "go") + tags, err := c.ListTags(ctx) + must(t, err) + check(t, "tags", tags[0].Name, "a") + added, err := c.AddTags(ctx, "d1", []string{"b"}) + must(t, err) + check(t, "added", added, []string{"a", "b"}) + remaining, err := c.RemoveTags(ctx, "d1", []string{"b"}) + must(t, err) + check(t, "remove query", (*seen)[4].Query["tags"], []string{"b"}) + check(t, "remaining", remaining, []string{"a"}) + graph, err := c.GetGraph(ctx, WithThreshold(0.5)) + must(t, err) + check(t, "threshold", (*seen)[5].Query["threshold"], []string{"0.5"}) + check(t, "graph", []any{graph.Nodes[0].Type, graph.Edges[0].Weight}, []any{"document", 1.0}) +} + +func TestOverviewAskSync(t *testing.T) { + c, seen := fakeServer(t, + reply{data: map[string]any{"stats": map[string]any{"totalDocuments": 3, "databaseSizeBytes": 4096}, "health": map[string]any{"database": "ok"}}}, + reply{data: map[string]any{"mode": "answer", "answer": "42", "sources": []any{map[string]any{"documentId": "d1", "title": "Doc", "score": 0.5}}}}, + reply{status: 202, data: map[string]any{"taskId": "t1", "operation": "sync", "status": "running"}}, + reply{status: 202, data: map[string]any{"taskId": "t2", "operation": "sync", "status": "running"}}, + reply{data: map[string]any{"taskId": "t1", "cancelRequested": true, "status": "cancelled"}}, + ) + overview, err := c.Overview(ctx) + must(t, err) + check(t, "documents", overview.Stats.TotalDocuments, int64(3)) + answer, err := c.Ask(ctx, "why?", WithTopK(3)) + must(t, err) + check(t, "ask body", (*seen)[1].Body, map[string]any{"question": "why?", "topK": float64(3)}) + check(t, "answer", []any{answer.Answer, answer.Sources[0].DocumentID}, []any{"42", "d1"}) + task, err := c.Sync(ctx, "notes") + must(t, err) + check(t, "sync", []any{task.Operation, (*seen)[2].Body}, []any{"sync", map[string]any{"name": "notes"}}) + _, err = c.SyncAll(ctx) + must(t, err) + check(t, "sync all", (*seen)[3].Body, map[string]any{"all": true}) + cancelled, err := c.CancelTask(ctx, "t1") + must(t, err) + check(t, "cancelled", cancelled, true) +} + +func TestErrors(t *testing.T) { + c, _ := fakeServer(t, + reply{status: 404, raw: `{"error":{"code":"DOCUMENT_NOT_FOUND","message":"gone"}}`}, + reply{status: 502, raw: `bad gateway`}, + ) + _, err := c.GetDocument(ctx, "x") + var apiErr *Error + if !errors.As(err, &apiErr) || apiErr.Code != "DOCUMENT_NOT_FOUND" || apiErr.Message != "gone" { + t.Fatalf("expected API error, got %v", err) } - if apiErr.StatusCode != 500 { - t.Errorf("expected status 500, got %d", apiErr.StatusCode) + if !errors.Is(err, ErrNotFound) { + t.Errorf("expected errors.Is(err, ErrNotFound)") } - if apiErr.Code != "INTERNAL_ERROR" { - t.Errorf("expected code INTERNAL_ERROR, got %q", apiErr.Code) + _, err = c.Health(ctx) + if !errors.As(err, &apiErr) || apiErr.StatusCode != 502 || apiErr.Message != "bad gateway" { + t.Fatalf("expected raw error body, got %v", err) } } diff --git a/sdk/go/connectors.go b/sdk/go/connectors.go index 706d6b6b..495afc51 100644 --- a/sdk/go/connectors.go +++ b/sdk/go/connectors.go @@ -1,4 +1,14 @@ package libscope -// Connector operations will be added as the REST API exposes connector sync endpoints. -// This file is a placeholder for future connector sync operations. +import "context" + +// Sync starts syncing a saved connector connection (created with "libscope connect") and +// returns the background task. +func (c *Client) Sync(ctx context.Context, name string) (*Task, error) { + return c.startTask(ctx, "/sync", map[string]any{"name": name}) +} + +// SyncAll starts syncing every saved connection and returns the background task. +func (c *Client) SyncAll(ctx context.Context) (*Task, error) { + return c.startTask(ctx, "/sync", map[string]any{"all": true}) +} diff --git a/sdk/go/documents.go b/sdk/go/documents.go index 782f5cf2..2b55eb64 100644 --- a/sdk/go/documents.go +++ b/sdk/go/documents.go @@ -1,166 +1,39 @@ package libscope import ( - "bytes" "context" - "encoding/json" - "fmt" "net/http" - "net/url" - "strconv" ) -// AddDocumentOption configures AddDocument. -type AddDocumentOption func(map[string]interface{}) +// documentsPath is the REST collection of documents. +const documentsPath = "/documents" -// WithDocumentTopic sets the topic for a new document. -func WithDocumentTopic(topic string) AddDocumentOption { - return func(m map[string]interface{}) { - m["topic"] = topic - } +// AddText starts adding a document from text and returns the background task. Params: +// WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags, WithURL. +// Use WaitForTask and Task.DecodeResult (*IngestResult) to get the new document. +func (c *Client) AddText(ctx context.Context, title, content string, params ...Param) (*Task, error) { + body := fields(map[string]any{"title": title, "content": content}, params) + return c.startTask(ctx, documentsPath, body) } -// WithDocumentTags sets the tags for a new document. -func WithDocumentTags(tags ...string) AddDocumentOption { - return func(m map[string]interface{}) { - m["tags"] = tags - } +// AddURL starts adding a web page, a crawled site (WithSpider, WithMaxPages) or a public +// GitHub/GitLab repository, and returns the background task. +func (c *Client) AddURL(ctx context.Context, url string, params ...Param) (*Task, error) { + return c.startTask(ctx, documentsPath, fields(map[string]any{"url": url}, params)) } -// AddDocument indexes a document from a URL. -func (c *Client) AddDocument(ctx context.Context, docURL string, opts ...AddDocumentOption) (*Document, error) { - body := map[string]interface{}{ - "url": docURL, - } - for _, opt := range opts { - opt(body) - } - - data, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("libscope: encoding request: %w", err) - } - - resp, err := c.do(ctx, http.MethodPost, "/api/v1/documents/url", bytes.NewReader(data)) - if err != nil { - return nil, err - } - result, err := decodeResponse[Document](resp) - if err != nil { - return nil, err - } - return &result, nil -} - -// AddTextOption configures AddText. -type AddTextOption func(map[string]interface{}) - -// WithTextTopic sets the topic for a text document. -func WithTextTopic(topic string) AddTextOption { - return func(m map[string]interface{}) { - m["topic"] = topic - } -} - -// WithTextTags sets the tags for a text document. -func WithTextTags(tags ...string) AddTextOption { - return func(m map[string]interface{}) { - m["tags"] = tags - } -} - -// WithTextSource sets the source type for a text document. -func WithTextSource(source string) AddTextOption { - return func(m map[string]interface{}) { - m["source"] = source - } -} - -// AddText indexes a document from raw text content. -func (c *Client) AddText(ctx context.Context, title, content string, opts ...AddTextOption) (*Document, error) { - body := map[string]interface{}{ - "title": title, - "content": content, - } - for _, opt := range opts { - opt(body) - } - - data, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("libscope: encoding request: %w", err) - } - - resp, err := c.do(ctx, http.MethodPost, "/api/v1/documents", bytes.NewReader(data)) - if err != nil { - return nil, err - } - result, err := decodeResponse[Document](resp) - if err != nil { - return nil, err - } - return &result, nil -} - -// GetDocument retrieves a single document by ID. -func (c *Client) GetDocument(ctx context.Context, id string) (*Document, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/documents/"+url.PathEscape(id), nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[Document](resp) - if err != nil { - return nil, err - } - return &result, nil +// GetDocument returns a document with its content, tags, links and rating summary. +func (c *Client) GetDocument(ctx context.Context, documentID string) (*DocumentView, error) { + return get[DocumentView](ctx, c, documentsPath+segment(documentID), nil) } -// ListOption configures ListDocuments. -type ListOption func(*url.Values) - -// WithListTopic filters documents by topic. -func WithListTopic(topic string) ListOption { - return func(v *url.Values) { - v.Set("topic", topic) - } -} - -// WithListLimit limits the number of documents returned. -func WithListLimit(n int) ListOption { - return func(v *url.Values) { - v.Set("limit", strconv.Itoa(n)) - } -} - -// ListDocuments lists documents with optional filters. -func (c *Client) ListDocuments(ctx context.Context, opts ...ListOption) ([]Document, error) { - params := url.Values{} - for _, opt := range opts { - opt(¶ms) - } - - path := "/api/v1/documents" - if len(params) > 0 { - path += "?" + params.Encode() - } - - resp, err := c.do(ctx, http.MethodGet, path, nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[[]Document](resp) - if err != nil { - return nil, err - } - return result, nil +// ListDocuments lists documents, newest first. Params: WithLimit, WithOffset and the filters +// WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags. +func (c *Client) ListDocuments(ctx context.Context, params ...Param) (*Page[Document], error) { + return get[Page[Document]](ctx, c, documentsPath, query(fields(map[string]any{}, params))) } -// DeleteDocument deletes a document by ID. -func (c *Client) DeleteDocument(ctx context.Context, id string) error { - resp, err := c.do(ctx, http.MethodDelete, "/api/v1/documents/"+url.PathEscape(id), nil) - if err != nil { - return err - } - _, err = decodeResponse[map[string]interface{}](resp) - return err +// DeleteDocument deletes a document. +func (c *Client) DeleteDocument(ctx context.Context, documentID string) error { + return c.call(ctx, http.MethodDelete, documentsPath+segment(documentID), nil, nil, nil) } diff --git a/sdk/go/documents_test.go b/sdk/go/documents_test.go deleted file mode 100644 index 6c18e7e8..00000000 --- a/sdk/go/documents_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package libscope - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" -) - -const ( - errUnexpectedPath = "unexpected path: %s" - headerContentType = "Content-Type" - mimeJSON = "application/json" - errUnexpected = "unexpected error: %v" - testGoDevDocURL = "https://go.dev/doc/" -) - -func TestAddText(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - t.Errorf("expected POST, got %s", r.Method) - } - if r.URL.Path != "/api/v1/documents" { - t.Errorf(errUnexpectedPath, r.URL.Path) - } - - var body map[string]interface{} - json.NewDecoder(r.Body).Decode(&body) - if body["title"] != "Test Doc" { - t.Errorf("expected title 'Test Doc', got %v", body["title"]) - } - if body["content"] != "hello world" { - t.Errorf("expected content 'hello world', got %v", body["content"]) - } - - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(201) - w.Write([]byte(`{"data":{"id":"doc-1","title":"Test Doc","source":"manual"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - doc, err := c.AddText(context.Background(), "Test Doc", "hello world") - if err != nil { - t.Fatalf(errUnexpected, err) - } - if doc.ID != "doc-1" { - t.Errorf("expected id doc-1, got %q", doc.ID) - } -} - -func TestAddTextWithOptions(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var body map[string]interface{} - json.NewDecoder(r.Body).Decode(&body) - if body["topic"] != "golang" { - t.Errorf("expected topic 'golang', got %v", body["topic"]) - } - tags, ok := body["tags"].([]interface{}) - if !ok || len(tags) != 2 { - t.Errorf("expected 2 tags, got %v", body["tags"]) - } - - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(201) - w.Write([]byte(`{"data":{"id":"doc-2","title":"Go Guide","topic":"golang","tags":["go","tutorial"]}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - doc, err := c.AddText(context.Background(), "Go Guide", "content", - WithTextTopic("golang"), - WithTextTags("go", "tutorial"), - ) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if doc.Topic != "golang" { - t.Errorf("expected topic golang, got %q", doc.Topic) - } -} - -func TestAddDocument(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/v1/documents/url" { - t.Errorf(errUnexpectedPath, r.URL.Path) - } - var body map[string]interface{} - json.NewDecoder(r.Body).Decode(&body) - if body["url"] != testGoDevDocURL { - t.Errorf("expected url, got %v", body["url"]) - } - - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(201) - w.Write([]byte(`{"data":{"id":"doc-3","title":"Go Documentation","url":"` + testGoDevDocURL + `"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - doc, err := c.AddDocument(context.Background(), testGoDevDocURL) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if doc.URL != testGoDevDocURL { - t.Errorf("expected url, got %q", doc.URL) - } -} - -func TestGetDocument(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/v1/documents/doc-1" { - t.Errorf(errUnexpectedPath, r.URL.Path) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"id":"doc-1","title":"Test"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - doc, err := c.GetDocument(context.Background(), "doc-1") - if err != nil { - t.Fatalf(errUnexpected, err) - } - if doc.ID != "doc-1" { - t.Errorf("expected doc-1, got %q", doc.ID) - } -} - -func TestGetDocumentNotFound(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(404) - w.Write([]byte(`{"error":{"code":"NOT_FOUND","message":"Document not found"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - _, err := c.GetDocument(context.Background(), "nonexistent") - if err == nil { - t.Fatal("expected error") - } - apiErr, ok := err.(*Error) - if !ok { - t.Fatalf("expected *Error, got %T", err) - } - if apiErr.StatusCode != 404 { - t.Errorf("expected 404, got %d", apiErr.StatusCode) - } -} - -func TestListDocuments(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("topic") != "go" { - t.Errorf("expected topic=go, got %q", r.URL.Query().Get("topic")) - } - if r.URL.Query().Get("limit") != "10" { - t.Errorf("expected limit=10, got %q", r.URL.Query().Get("limit")) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":[{"id":"doc-1","title":"Doc 1"},{"id":"doc-2","title":"Doc 2"}]}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - docs, err := c.ListDocuments(context.Background(), WithListTopic("go"), WithListLimit(10)) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if len(docs) != 2 { - t.Errorf("expected 2 docs, got %d", len(docs)) - } -} - -func TestDeleteDocument(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodDelete { - t.Errorf("expected DELETE, got %s", r.Method) - } - if r.URL.Path != "/api/v1/documents/doc-1" { - t.Errorf(errUnexpectedPath, r.URL.Path) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"deleted":true}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - err := c.DeleteDocument(context.Background(), "doc-1") - if err != nil { - t.Fatalf(errUnexpected, err) - } -} diff --git a/sdk/go/errors.go b/sdk/go/errors.go index 9c770549..2ba9ca6f 100644 --- a/sdk/go/errors.go +++ b/sdk/go/errors.go @@ -1,8 +1,11 @@ package libscope -import "fmt" +import ( + "encoding/json" + "fmt" +) -// Error represents an API error returned by libscope. +// Error is an error response from the libscope API. type Error struct { StatusCode int Code string @@ -16,8 +19,41 @@ func (e *Error) Error() string { return fmt.Sprintf("libscope: HTTP %d: %s", e.StatusCode, e.Message) } +// Is matches errors with the same HTTP status, so errors.Is(err, ErrNotFound) works. +func (e *Error) Is(target error) bool { + t, ok := target.(*Error) + return ok && t.StatusCode == e.StatusCode +} + var ( ErrNotFound = &Error{StatusCode: 404, Message: "not found"} ErrBadRequest = &Error{StatusCode: 400, Message: "bad request"} ErrServerError = &Error{StatusCode: 500, Message: "server error"} ) + +// newError builds an *Error from an error response body. +func newError(status int, body []byte) *Error { + var envelope struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if json.Unmarshal(body, &envelope) == nil && envelope.Error.Message != "" { + return &Error{StatusCode: status, Code: envelope.Error.Code, Message: envelope.Error.Message} + } + return &Error{StatusCode: status, Message: string(body)} +} + +// TaskError is returned by WaitForTask when the task failed or was cancelled. +type TaskError struct { + Task *Task +} + +func (e *TaskError) Error() string { + reason := e.Task.Error + if reason == "" { + reason = e.Task.Status + } + return fmt.Sprintf("libscope: task %s (%s) %s", e.Task.ID, e.Task.Operation, reason) +} diff --git a/sdk/go/examples/quickstart/main.go b/sdk/go/examples/quickstart/main.go index 2d1bd5b0..032cfc6e 100644 --- a/sdk/go/examples/quickstart/main.go +++ b/sdk/go/examples/quickstart/main.go @@ -1,42 +1,47 @@ +// Quick-start example for the libscope Go SDK. Start the server first: libscope serve api. package main import ( "context" "fmt" "log" + "time" libscope "github.com/RobertLD/libscope/sdk/go" ) func main() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() client := libscope.NewClient() - // Add a document from a URL - doc, err := client.AddDocument(context.Background(), "https://go.dev/doc/") + // Adding runs on the server as a background task; wait for it to get the new documents. + task, err := client.AddText(ctx, "Go Concurrency", + "Goroutines are lightweight threads managed by the Go runtime.", + libscope.WithTags("go", "concurrency"), + ) if err != nil { log.Fatal(err) } - fmt.Printf("Indexed: %s (%s)\n", doc.Title, doc.ID) - - // Add a text document - textDoc, err := client.AddText(context.Background(), "Go Concurrency", - "Goroutines are lightweight threads managed by the Go runtime.", - libscope.WithTextTags("go", "concurrency"), - ) + task, err = client.WaitForTask(ctx, task.ID, time.Second) if err != nil { log.Fatal(err) } - fmt.Printf("Added: %s (%s)\n", textDoc.Title, textDoc.ID) + var added libscope.IngestResult + if err := task.DecodeResult(&added); err != nil { + log.Fatal(err) + } + for _, doc := range added.Documents { + fmt.Printf("Added: %s (%s)\n", doc.Title, doc.DocumentID) + } // Search - results, err := client.Search(context.Background(), "goroutines", - libscope.WithLimit(5), - ) + page, err := client.Search(ctx, "goroutines", libscope.WithLimit(5)) if err != nil { log.Fatal(err) } - fmt.Printf("\nSearch results for 'goroutines' (%d total):\n", results.Total) - for _, hit := range results.Results { - fmt.Printf(" %s: %.2f\n", hit.Document.Title, hit.Score) + fmt.Printf("\nSearch results for 'goroutines' (%d total):\n", page.Total) + for _, hit := range page.Items { + fmt.Printf(" %s: %.2f\n", hit.Title, hit.Score) } } diff --git a/sdk/go/graph.go b/sdk/go/graph.go index 63b6f2c6..7ed96df6 100644 --- a/sdk/go/graph.go +++ b/sdk/go/graph.go @@ -1,4 +1,8 @@ package libscope -// Knowledge graph operations will be added as the REST API exposes graph endpoints. -// This file is a placeholder for future knowledge graph operations. +import "context" + +// GetGraph returns the knowledge graph. Params: WithTopic, WithTag, WithThreshold, WithMaxNodes. +func (c *Client) GetGraph(ctx context.Context, params ...Param) (*Graph, error) { + return get[Graph](ctx, c, "/graph", query(fields(map[string]any{}, params))) +} diff --git a/sdk/go/models.go b/sdk/go/models.go index 1547f0f6..386d60bc 100644 --- a/sdk/go/models.go +++ b/sdk/go/models.go @@ -1,81 +1,192 @@ package libscope -// Document represents an indexed document in libscope. +import "encoding/json" + +// Page is one page of a list result. +type Page[T any] struct { + Items []T `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +// Document is a document without its content (list results). type Document struct { - ID string `json:"id"` - Title string `json:"title"` - URL string `json:"url,omitempty"` - Topic string `json:"topic,omitempty"` - TopicID string `json:"topicId,omitempty"` - Tags []string `json:"tags,omitempty"` - ContentHash string `json:"content_hash,omitempty"` - Source string `json:"source,omitempty"` - CreatedAt string `json:"created_at,omitempty"` - UpdatedAt string `json:"updated_at,omitempty"` -} - -// SearchResult holds the result of a search query. -type SearchResult struct { - Results []SearchHit `json:"results"` - Total int `json:"total"` - Query string `json:"query"` -} - -// SearchHit represents a single search match. + DocumentID string `json:"documentId"` + Title string `json:"title"` + SourceType string `json:"sourceType"` + Library string `json:"library,omitempty"` + Version string `json:"version,omitempty"` + TopicID string `json:"topicId,omitempty"` + URL string `json:"url,omitempty"` + ContentHash string `json:"contentHash,omitempty"` + SubmittedBy string `json:"submittedBy,omitempty"` + ContentLength int `json:"contentLength,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + UpdatedAt string `json:"updatedAt,omitempty"` +} + +// DocumentView is a document with (a page of) its content, tags, links and ratings. +type DocumentView struct { + Document Document `json:"document"` + Content string `json:"content"` + ContentLength int `json:"contentLength"` + Offset int `json:"offset"` + NextOffset *int `json:"nextOffset"` + Tags []string `json:"tags"` + Links map[string]any `json:"links"` + Ratings RatingSummary `json:"ratings"` +} + +// RatingSummary summarizes a document's ratings. +type RatingSummary struct { + AverageRating float64 `json:"averageRating"` + TotalRatings int `json:"totalRatings"` + Corrections int `json:"corrections"` +} + +// SearchHit is a matching chunk. type SearchHit struct { - Document Document `json:"document"` - Score float64 `json:"score"` - ChunkText string `json:"chunk_text"` + DocumentID string `json:"documentId"` + ChunkID string `json:"chunkId"` + Title string `json:"title"` + Content string `json:"content"` + SourceType string `json:"sourceType"` + Library string `json:"library,omitempty"` + Version string `json:"version,omitempty"` + TopicID string `json:"topicId,omitempty"` + URL string `json:"url,omitempty"` + Score float64 `json:"score"` + AvgRating *float64 `json:"avgRating,omitempty"` } -// Topic represents a topic grouping for documents. +// Topic groups documents. type Topic struct { - ID string `json:"id"` - Name string `json:"name"` - ParentID string `json:"parentId,omitempty"` + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + ParentID string `json:"parentId,omitempty"` + DocumentCount int `json:"documentCount,omitempty"` } -// Tag represents a tag applied to documents. +// Tag is a tag with its document count. type Tag struct { - ID string `json:"id,omitempty"` - Name string `json:"name"` + ID string `json:"id"` + Name string `json:"name"` + DocumentCount int `json:"documentCount"` } -// Stats holds analytics/statistics about the libscope instance. +// Stats holds the counts of an Overview. type Stats struct { - TotalDocuments int64 `json:"totalDocuments"` - DatabaseSize int64 `json:"databaseSizeBytes"` + TotalDocuments int64 `json:"totalDocuments"` + TotalChunks int64 `json:"totalChunks"` + TotalTopics int64 `json:"totalTopics"` + DatabaseSizeBytes int64 `json:"databaseSizeBytes"` } -// HealthStatus holds the health check response. +// Overview holds counts, topics, installed packs, the embedding index and health. +type Overview struct { + Stats Stats `json:"stats"` + Topics []Topic `json:"topics"` + Packs []map[string]any `json:"packs"` + Index map[string]any `json:"index"` + Health map[string]any `json:"health"` +} + +// HealthStatus is the liveness check response. type HealthStatus struct { - Status string `json:"status"` - DocCount int64 `json:"docCount"` - DBSize int64 `json:"dbSize"` + Status string `json:"status"` } -// AskResult holds the response from the RAG ask endpoint. +// AskResult is an answer (Mode "answer"), or the context to answer from (Mode "context"). type AskResult struct { - Answer string `json:"answer"` - Sources []SearchHit `json:"sources,omitempty"` + Mode string `json:"mode"` + Answer string `json:"answer,omitempty"` + ContextPrompt string `json:"contextPrompt,omitempty"` + Sources []AskSource `json:"sources"` + Model string `json:"model,omitempty"` + TokensUsed int `json:"tokensUsed,omitempty"` +} + +// AskSource is a chunk an answer is based on. +type AskSource struct { + DocumentID string `json:"documentId"` + Title string `json:"title"` + Chunk string `json:"chunk"` + Score float64 `json:"score"` +} + +// Graph is the knowledge graph of documents, topics and tags. +type Graph struct { + Nodes []GraphNode `json:"nodes"` + Edges []GraphEdge `json:"edges"` +} + +// GraphNode is a document, topic or tag. +type GraphNode struct { + ID string `json:"id"` + Label string `json:"label"` + Type string `json:"type"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +// GraphEdge connects two nodes. +type GraphEdge struct { + Source string `json:"source"` + Target string `json:"target"` + Type string `json:"type"` + Weight float64 `json:"weight"` +} + +// Task is a background task on the server (adding documents, syncing connections). +type Task struct { + ID string `json:"id"` + Operation string `json:"operation,omitempty"` + Status string `json:"status"` + Progress *TaskProgress `json:"progress,omitempty"` + // Result is the operation result as JSON, once Status is "completed". See DecodeResult. + Result string `json:"result,omitempty"` + Error string `json:"error,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + CompletedAt string `json:"completedAt,omitempty"` +} + +// TaskProgress reports how far a task got. +type TaskProgress struct { + Current int `json:"current"` + Total int `json:"total"` + Message string `json:"message,omitempty"` } -// apiResponse is the standard success envelope from the REST API. -type apiResponse[T any] struct { - Data T `json:"data"` - Meta *apiMeta `json:"meta,omitempty"` +// Done reports whether the task completed, failed or was cancelled. +func (t *Task) Done() bool { + return t.Status == "completed" || t.Status == "failed" || t.Status == "cancelled" } -type apiMeta struct { - Took int `json:"took,omitempty"` +// DecodeResult decodes the task's result into v (for AddText and AddURL: *IngestResult). +func (t *Task) DecodeResult(v any) error { + return json.Unmarshal([]byte(t.Result), v) } -// apiErrorResponse is the standard error envelope from the REST API. -type apiErrorResponse struct { - Error apiErrorDetail `json:"error"` +// IngestResult is the result of an AddText or AddURL task. +type IngestResult struct { + Kind string `json:"kind"` + Documents []IngestedDocument `json:"documents"` + Errors []struct { + Source string `json:"source"` + Error string `json:"error"` + } `json:"errors"` + Skipped []struct { + Source string `json:"source"` + Reason string `json:"reason"` + } `json:"skipped"` } -type apiErrorDetail struct { - Code string `json:"code"` - Message string `json:"message"` +// IngestedDocument is one document added by a task. +type IngestedDocument struct { + DocumentID string `json:"documentId"` + Title string `json:"title"` + ChunkCount int `json:"chunkCount"` + Source string `json:"source,omitempty"` } diff --git a/sdk/go/params.go b/sdk/go/params.go new file mode 100644 index 00000000..031f59bc --- /dev/null +++ b/sdk/go/params.go @@ -0,0 +1,88 @@ +package libscope + +import ( + "fmt" + "net/url" +) + +// Param sets one optional request field. The same Params work for every method that +// accepts the field (for example WithTopic for Search, Ask, ListDocuments and AddText). +type Param func(map[string]any) + +func set(key string, value any) Param { + return func(m map[string]any) { m[key] = value } +} + +// WithTopic filters by topic (ID or name), or sets the topic of a new document. +func WithTopic(topic string) Param { return set("topic", topic) } + +// WithLibrary filters by library, or sets the library of a new document. +func WithLibrary(library string) Param { return set("library", library) } + +// WithVersion filters by library version, or sets the version of a new document. +func WithVersion(version string) Param { return set("version", version) } + +// WithSourceType filters by source type (manual, library, topic, model), or sets it. +func WithSourceType(sourceType string) Param { return set("sourceType", sourceType) } + +// WithTags keeps documents that carry all of these tags, or tags a new document. +func WithTags(tags ...string) Param { return set("tags", tags) } + +// WithMinRating keeps documents with at least this average rating (1-5). +func WithMinRating(rating float64) Param { return set("minRating", rating) } + +// WithLimit sets the page size. +func WithLimit(n int) Param { return set("limit", n) } + +// WithOffset skips this many results (paging). +func WithOffset(n int) Param { return set("offset", n) } + +// WithTopK sets how many chunks Ask retrieves. +func WithTopK(n int) Param { return set("topK", n) } + +// WithURL stores a source URL with inline content (AddText). +func WithURL(u string) Param { return set("url", u) } + +// WithSpider makes AddURL also crawl the pages the URL links to. +func WithSpider() Param { return set("spider", true) } + +// WithMaxPages limits a crawl (AddURL with WithSpider). +func WithMaxPages(n int) Param { return set("maxPages", n) } + +// WithParent sets the parent topic (ID or name) of a new topic. +func WithParent(parent string) Param { return set("parent", parent) } + +// WithDescription sets the description of a new topic. +func WithDescription(description string) Param { return set("description", description) } + +// WithTag limits the graph to documents with this tag. +func WithTag(tag string) Param { return set("tag", tag) } + +// WithThreshold sets the similarity threshold for graph edges (0-1). +func WithThreshold(t float64) Param { return set("threshold", t) } + +// WithMaxNodes limits the number of graph nodes. +func WithMaxNodes(n int) Param { return set("maxNodes", n) } + +// fields applies params over base. +func fields(base map[string]any, params []Param) map[string]any { + for _, p := range params { + p(base) + } + return base +} + +// query encodes fields as a query string; lists become repeated keys. +func query(m map[string]any) url.Values { + v := url.Values{} + for key, value := range m { + if list, ok := value.([]string); ok { + for _, item := range list { + v.Add(key, item) + } + continue + } + v.Set(key, fmt.Sprint(value)) + } + return v +} diff --git a/sdk/go/search.go b/sdk/go/search.go index 5ad95ea7..cd395047 100644 --- a/sdk/go/search.go +++ b/sdk/go/search.go @@ -3,89 +3,17 @@ package libscope import ( "context" "net/http" - "net/url" - "strconv" ) -// SearchOptions holds the configuration for a search query. -type SearchOptions struct { - Limit int - Topic string - Tags []string - MinScore float64 +// Search searches the knowledge base by meaning and keywords. Params: WithLimit, WithOffset, +// WithMinRating and the filters WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags. +func (c *Client) Search(ctx context.Context, q string, params ...Param) (*Page[SearchHit], error) { + return get[Page[SearchHit]](ctx, c, "/search", query(fields(map[string]any{"query": q}, params))) } -// SearchOption configures a search query. -type SearchOption func(*SearchOptions) - -// WithLimit sets the maximum number of results. -func WithLimit(n int) SearchOption { - return func(o *SearchOptions) { - o.Limit = n - } -} - -// WithTopic filters results by topic. -func WithTopic(t string) SearchOption { - return func(o *SearchOptions) { - o.Topic = t - } -} - -// WithTags filters results by tags. -func WithTags(tags ...string) SearchOption { - return func(o *SearchOptions) { - o.Tags = tags - } -} - -// WithMinScore filters results below a minimum score. -func WithMinScore(s float64) SearchOption { - return func(o *SearchOptions) { - o.MinScore = s - } -} - -// Search performs a semantic search across indexed documents. -func (c *Client) Search(ctx context.Context, query string, opts ...SearchOption) (*SearchResult, error) { - options := &SearchOptions{} - for _, opt := range opts { - opt(options) - } - - params := url.Values{} - params.Set("q", query) - if options.Limit > 0 { - params.Set("limit", strconv.Itoa(options.Limit)) - } - if options.Topic != "" { - params.Set("topic", options.Topic) - } - if len(options.Tags) > 0 { - // The API supports a single "tag" query param - params.Set("tag", options.Tags[0]) - } - - resp, err := c.do(ctx, http.MethodGet, "/api/v1/search?"+params.Encode(), nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[SearchResult](resp) - if err != nil { - return nil, err - } - - // Client-side min score filtering - if options.MinScore > 0 { - filtered := make([]SearchHit, 0, len(result.Results)) - for _, hit := range result.Results { - if hit.Score >= options.MinScore { - filtered = append(filtered, hit) - } - } - result.Results = filtered - result.Total = len(filtered) - } - - return &result, nil +// Ask answers a question from the knowledge base (the server needs an LLM). Params: WithTopK, +// WithMinRating and the filters WithTopic, WithLibrary, WithVersion, WithSourceType, WithTags. +func (c *Client) Ask(ctx context.Context, question string, params ...Param) (*AskResult, error) { + body := fields(map[string]any{"question": question}, params) + return send[AskResult](ctx, c, http.MethodPost, "/ask", body) } diff --git a/sdk/go/search_test.go b/sdk/go/search_test.go deleted file mode 100644 index d84e00a7..00000000 --- a/sdk/go/search_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package libscope - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" -) - -func TestSearch(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("q") != "goroutines" { - t.Errorf("expected q=goroutines, got %q", r.URL.Query().Get("q")) - } - if r.URL.Query().Get("limit") != "5" { - t.Errorf("expected limit=5, got %q", r.URL.Query().Get("limit")) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"results":[{"document":{"id":"doc-1","title":"Go Concurrency"},"score":0.95,"chunk_text":"goroutines are..."}],"total":1,"query":"goroutines"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - result, err := c.Search(context.Background(), "goroutines", WithLimit(5)) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if result.Total != 1 { - t.Errorf("expected 1 result, got %d", result.Total) - } - if result.Results[0].Score != 0.95 { - t.Errorf("expected score 0.95, got %f", result.Results[0].Score) - } -} - -func TestSearchWithTopic(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("topic") != "golang" { - t.Errorf("expected topic=golang, got %q", r.URL.Query().Get("topic")) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"results":[],"total":0,"query":"test"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - result, err := c.Search(context.Background(), "test", WithTopic("golang")) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if result.Total != 0 { - t.Errorf("expected 0 results, got %d", result.Total) - } -} - -func TestSearchWithTags(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("tag") != "tutorial" { - t.Errorf("expected tag=tutorial, got %q", r.URL.Query().Get("tag")) - } - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"results":[],"total":0,"query":"test"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - _, err := c.Search(context.Background(), "test", WithTags("tutorial")) - if err != nil { - t.Fatalf(errUnexpected, err) - } -} - -func TestSearchWithMinScore(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentType, mimeJSON) - w.Write([]byte(`{"data":{"results":[{"document":{"id":"d1","title":"High"},"score":0.9,"chunk_text":"high"},{"document":{"id":"d2","title":"Low"},"score":0.3,"chunk_text":"low"}],"total":2,"query":"test"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - result, err := c.Search(context.Background(), "test", WithMinScore(0.5)) - if err != nil { - t.Fatalf(errUnexpected, err) - } - if len(result.Results) != 1 { - t.Errorf("expected 1 result after min score filter, got %d", len(result.Results)) - } - if result.Results[0].Document.ID != "d1" { - t.Errorf("expected d1, got %q", result.Results[0].Document.ID) - } -} - -func TestSearchError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set(headerContentType, mimeJSON) - w.WriteHeader(400) - w.Write([]byte(`{"error":{"code":"VALIDATION_ERROR","message":"Query parameter 'q' is required"}}`)) - })) - defer srv.Close() - - c := NewClient(WithBaseURL(srv.URL)) - _, err := c.Search(context.Background(), "") - if err == nil { - t.Fatal("expected error") - } - apiErr, ok := err.(*Error) - if !ok { - t.Fatalf("expected *Error, got %T", err) - } - if apiErr.StatusCode != 400 { - t.Errorf("expected 400, got %d", apiErr.StatusCode) - } -} diff --git a/sdk/go/tags.go b/sdk/go/tags.go index 26bc441f..5d338be4 100644 --- a/sdk/go/tags.go +++ b/sdk/go/tags.go @@ -1,45 +1,40 @@ package libscope import ( - "bytes" "context" - "encoding/json" - "fmt" "net/http" "net/url" ) -// ListTags lists all tags. +type documentTags struct { + Tags []string `json:"tags"` +} + +// ListTags lists tags with their document counts. func (c *Client) ListTags(ctx context.Context) ([]Tag, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/tags", nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[[]Tag](resp) + page, err := get[Page[Tag]](ctx, c, "/tags", nil) if err != nil { return nil, err } - return result, nil + return page.Items, nil } -// AddTagsToDocument adds tags to a document. -func (c *Client) AddTagsToDocument(ctx context.Context, docID string, tags []string) ([]Tag, error) { - body := map[string]interface{}{ - "tags": tags, - } - data, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("libscope: encoding request: %w", err) - } - - path := "/api/v1/documents/" + url.PathEscape(docID) + "/tags" - resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(data)) +// AddTags adds tags to a document and returns the document's tags. +func (c *Client) AddTags(ctx context.Context, documentID string, tags []string) ([]string, error) { + path := documentsPath + segment(documentID) + "/tags" + out, err := send[documentTags](ctx, c, http.MethodPost, path, documentTags{Tags: tags}) if err != nil { return nil, err } - result, err := decodeResponse[[]Tag](resp) - if err != nil { + return out.Tags, nil +} + +// RemoveTags removes tags from a document and returns the document's remaining tags. +func (c *Client) RemoveTags(ctx context.Context, documentID string, tags []string) ([]string, error) { + var out documentTags + path := documentsPath + segment(documentID) + "/tags" + if err := c.call(ctx, http.MethodDelete, path, url.Values{"tags": tags}, nil, &out); err != nil { return nil, err } - return result, nil + return out.Tags, nil } diff --git a/sdk/go/tasks.go b/sdk/go/tasks.go new file mode 100644 index 00000000..a6d0b26e --- /dev/null +++ b/sdk/go/tasks.go @@ -0,0 +1,61 @@ +package libscope + +import ( + "context" + "net/http" + "time" +) + +// startTask posts body to a long-running route (answered with 202) and returns the task. +func (c *Client) startTask(ctx context.Context, path string, body any) (*Task, error) { + started, err := send[struct { + TaskID string `json:"taskId"` + Operation string `json:"operation"` + Status string `json:"status"` + }](ctx, c, http.MethodPost, path, body) + if err != nil { + return nil, err + } + return &Task{ID: started.TaskID, Operation: started.Operation, Status: started.Status}, nil +} + +// GetTask returns the status, progress and result of a background task. +func (c *Client) GetTask(ctx context.Context, taskID string) (*Task, error) { + return get[Task](ctx, c, "/tasks"+segment(taskID), nil) +} + +// CancelTask requests cancellation. It reports whether the task was still pending or running. +func (c *Client) CancelTask(ctx context.Context, taskID string) (bool, error) { + var out struct { + CancelRequested bool `json:"cancelRequested"` + } + path := "/tasks" + segment(taskID) + "/cancel" + if err := c.call(ctx, http.MethodPost, path, nil, nil, &out); err != nil { + return false, err + } + return out.CancelRequested, nil +} + +// WaitForTask polls a task every interval until it finishes or ctx is done. It returns +// *TaskError when the task failed or was cancelled. Use context.WithTimeout to bound the wait. +func (c *Client) WaitForTask(ctx context.Context, taskID string, interval time.Duration) (*Task, error) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + task, err := c.GetTask(ctx, taskID) + if err != nil { + return nil, err + } + if task.Done() { + if task.Status != "completed" { + return task, &TaskError{Task: task} + } + return task, nil + } + select { + case <-ctx.Done(): + return task, ctx.Err() + case <-ticker.C: + } + } +} diff --git a/sdk/go/topics.go b/sdk/go/topics.go index 78d8ba02..80e53f80 100644 --- a/sdk/go/topics.go +++ b/sdk/go/topics.go @@ -1,57 +1,20 @@ package libscope import ( - "bytes" "context" - "encoding/json" - "fmt" "net/http" ) -// CreateTopicOption configures CreateTopic. -type CreateTopicOption func(map[string]interface{}) - -// WithParentID sets the parent topic ID. -func WithParentID(id string) CreateTopicOption { - return func(m map[string]interface{}) { - m["parentId"] = id - } -} - -// ListTopics lists all topics. +// ListTopics lists topics with their document counts. func (c *Client) ListTopics(ctx context.Context) ([]Topic, error) { - resp, err := c.do(ctx, http.MethodGet, "/api/v1/topics", nil) - if err != nil { - return nil, err - } - result, err := decodeResponse[[]Topic](resp) + page, err := get[Page[Topic]](ctx, c, "/topics", nil) if err != nil { return nil, err } - return result, nil + return page.Items, nil } -// CreateTopic creates a new topic. -func (c *Client) CreateTopic(ctx context.Context, name string, opts ...CreateTopicOption) (*Topic, error) { - body := map[string]interface{}{ - "name": name, - } - for _, opt := range opts { - opt(body) - } - - data, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("libscope: encoding request: %w", err) - } - - resp, err := c.do(ctx, http.MethodPost, "/api/v1/topics", bytes.NewReader(data)) - if err != nil { - return nil, err - } - result, err := decodeResponse[Topic](resp) - if err != nil { - return nil, err - } - return &result, nil +// CreateTopic creates a topic. Params: WithParent, WithDescription. +func (c *Client) CreateTopic(ctx context.Context, name string, params ...Param) (*Topic, error) { + return send[Topic](ctx, c, http.MethodPost, "/topics", fields(map[string]any{"name": name}, params)) } diff --git a/sdk/python/README.md b/sdk/python/README.md index c43a1208..86640d5b 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -1,40 +1,43 @@ # libscope Python SDK -Python client library for the [libscope](https://github.com/RobertLD/libscope) AI knowledge base. +Python client for the [libscope](https://github.com/RobertLD/libscope) REST API (`/api/v1`). ## Installation ```bash -pip install libscope +pip install pylibscope ``` -Or for development: +For development: ```bash pip install -e ".[dev]" +pytest ``` +Start a server with `libscope serve api` (default `http://localhost:3378`). + ## Quick Start ```python -from libscope import LibscopeClient +from pylibscope import LibscopeClient with LibscopeClient() as client: - # Add a document from a URL - doc = client.add_document("https://docs.python.org/3/tutorial/") + # Adding runs as a background task on the server. wait_for_task polls it. + task = client.add_url("https://docs.python.org/3/tutorial/") + done = client.wait_for_task(task.id) + doc_id = done.result["documents"][0]["documentId"] - # Add a document from raw text + client.create_topic("python") # a topic must exist before documents use it client.add_text("My Notes", "Some useful content...", topic="python") - # Search - results = client.search("how to use decorators") - for hit in results.results: + page = client.search("how to use decorators", limit=5) + for hit in page.items: print(f"{hit.title}: {hit.score:.2f}") - # Manage tags - client.add_tags(doc.id, ["python", "tutorial"]) + client.add_tags(doc_id, ["python", "tutorial"]) - # Ask a question (RAG) + # ask needs an LLM on the server (otherwise it raises ServerError with the setting to change) answer = client.ask("What is the best practice for error handling?") print(answer.answer) ``` @@ -43,68 +46,80 @@ with LibscopeClient() as client: ```python import asyncio -from libscope import AsyncLibscopeClient +from pylibscope import AsyncLibscopeClient async def main(): async with AsyncLibscopeClient() as client: - results = await client.search("decorators") - for hit in results.results: + page = await client.search("decorators") + for hit in page.items: print(f"{hit.title}: {hit.score:.2f}") asyncio.run(main()) ``` +`AsyncLibscopeClient` has the same methods as `LibscopeClient`, except that its `wait_for_task` has no `timeout` parameter. Set the time limit around the call: + +```python +task = await client.add_url("https://example.com/docs") +done = await asyncio.wait_for(client.wait_for_task(task.id), timeout=300) +``` + +On Python 3.11 and later, `async with asyncio.timeout(300):` also works. At the limit, `asyncio.wait_for` raises `asyncio.TimeoutError` (the same class as `TimeoutError` on Python 3.11 and later). + ## Configuration ```python -# Custom server URL and timeout client = LibscopeClient( base_url="http://my-server:3378", timeout=60.0, + api_key="...", # sent as "Authorization: Bearer ..." when the server sets LIBSCOPE_API_KEY ) ``` +## Background tasks + +`add_text`, `add_url`, `sync` and `sync_all` return a `Task` at once (the server answers `202`). Call `wait_for_task(task.id, timeout=300, interval=1.0)` to poll `GET /api/v1/tasks/:taskId` until the task finishes. It returns the task, with `result` holding the operation result. It raises `TaskFailedError` when the task failed or was cancelled, and `TimeoutError` after `timeout` seconds. The async `wait_for_task(task.id, interval=1.0)` has no `timeout` (see [Async Usage](#async-usage)). `get_task` and `cancel_task` are also available. + ## API Reference -### Document Operations +Filters: `topic`, `library`, `version`, `source_type`, `tags` (a document must have all of them). -| Method | Description | -| ------------------------------------------------- | ------------------------- | -| `search(query, *, limit, topic, tags, min_score)` | Semantic search | -| `add_document(url, *, topic, tags)` | Index a document from URL | -| `add_text(title, content, *, topic, tags)` | Index raw text | -| `get_document(doc_id)` | Get a document by ID | -| `list_documents(*, topic, limit, offset)` | List documents | -| `delete_document(doc_id)` | Delete a document | +### Documents and search -### Topic Operations +| Method | Description | +| -------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| `search(query, *, limit, offset, min_rating, **filters)` | Search by meaning and keywords. Returns `Page[SearchHit]` | +| `ask(question, *, top_k, min_rating, **filters)` | Answer a question from the knowledge base (needs an LLM on the server) | +| `add_text(title, content, *, url, **filters)` | Start adding a document from text. Returns `Task` | +| `add_url(url, *, spider, max_pages, **filters)` | Start adding a web page, a crawled site or a public GitHub/GitLab repo. Returns `Task` | +| `get_document(document_id)` | Document with content, tags, links and ratings (`DocumentView`) | +| `list_documents(*, limit, offset, **filters)` | List documents. Returns `Page[Document]` | +| `delete_document(document_id)` | Delete a document | -| Method | Description | -| ---------------------------------- | --------------- | -| `list_topics()` | List all topics | -| `create_topic(name, *, parent_id)` | Create a topic | +### Topics, tags and graph -### Tag Operations +| Method | Description | +| ------------------------------------------------ | ---------------------------------------- | +| `list_topics()` | Topics with document counts | +| `create_topic(name, *, parent, description)` | Create a topic | +| `add_tags(document_id, tags)` | Add tags. Returns the document's tags | +| `remove_tags(document_id, tags)` | Remove tags. Returns the document's tags | +| `list_tags()` | Tags with document counts | +| `get_graph(*, topic, tag, threshold, max_nodes)` | Knowledge graph | -| Method | Description | -| --------------------------- | --------------------------- | -| `add_tags(doc_id, tags)` | Add tags to a document | -| `remove_tags(doc_id, tags)` | Remove tags from a document | -| `list_tags()` | List all tags | +### Connectors, tasks and server -### Analytics & Q&A +| Method | Description | +| ---------------------------------------------- | ------------------------------------------------- | +| `sync(name)` / `sync_all()` | Start syncing saved connections. Returns `Task` | +| `get_task(task_id)` / `cancel_task(task_id)` | Task status / request cancellation | +| `wait_for_task(task_id, *, timeout, interval)` | Poll until the task finishes | +| `overview()` | Counts, topics, installed packs, index and health | +| `health()` | Liveness check | -| Method | Description | -| ------------------------- | ------------------------------ | -| `get_analytics()` | Get knowledge base statistics | -| `ask(question, *, topic)` | RAG-powered question answering | -| `health()` | Check server health | +Errors: `NotFoundError` (404), `ValidationError` (400), `ServerError` (5xx), `LibscopeConnectionError`, `TaskFailedError`, and `LibscopeError` (base class; `code` holds the server's error code). ## Requirements - Python 3.9+ -- A running libscope server (default: `http://localhost:3378`) - -## License - -MIT +- A running libscope server (`libscope serve api`) diff --git a/sdk/python/examples/quickstart.py b/sdk/python/examples/quickstart.py index ada3448b..a9cb1c44 100644 --- a/sdk/python/examples/quickstart.py +++ b/sdk/python/examples/quickstart.py @@ -1,38 +1,33 @@ -"""Quick-start example for the libscope Python SDK.""" +"""Quick-start example for the libscope Python SDK. + +Start the server first: ``libscope serve api`` (http://localhost:3378). +""" from pylibscope import LibscopeClient -# Connect to a local libscope server (default: http://localhost:3378) with LibscopeClient() as client: - # Add a document from a URL - doc = client.add_document("https://docs.python.org/3/tutorial/") - print(f"Indexed: {doc.title} (id={doc.id})") + # Adding runs on the server as a background task; wait for it to get the new documents. + task = client.add_url("https://docs.python.org/3/tutorial/", tags=["python"]) + done = client.wait_for_task(task.id) + for doc in done.result["documents"]: + print(f"Indexed: {doc['title']} ({doc['documentId']})") - # Add a document from raw text - text_doc = client.add_text( + task = client.add_text( "Error Handling Guide", "Use try/except blocks to handle exceptions in Python...", - topic="python", tags=["tutorial", "errors"], ) + client.wait_for_task(task.id) # Search the knowledge base - results = client.search("how to use decorators") - for hit in results.results: + page = client.search("how to handle exceptions", limit=5) + for hit in page.items: print(f" {hit.title}: {hit.score:.2f}") - # Manage tags - client.add_tags(doc.id, ["python", "tutorial"]) - - # List topics - topics = client.list_topics() - for t in topics: - print(f"Topic: {t.name}") - - # Get analytics - stats = client.get_analytics() - print(f"Total documents: {stats.total_documents}") + # Counts, topics and index health + overview = client.overview() + print(f"Total documents: {overview.stats.total_documents}") - # Ask a question (RAG) + # Ask a question (the server needs an LLM: see llm.provider) answer = client.ask("What is the recommended error handling pattern?") print(f"Answer: {answer.answer}") diff --git a/sdk/python/src/pylibscope/__init__.py b/sdk/python/src/pylibscope/__init__.py index e1076723..33ca7618 100644 --- a/sdk/python/src/pylibscope/__init__.py +++ b/sdk/python/src/pylibscope/__init__.py @@ -6,39 +6,49 @@ LibscopeError, NotFoundError, ServerError, + TaskFailedError, ValidationError, ) from pylibscope.models import ( - Analytics, AskResult, + AskSource, Document, + DocumentView, Graph, GraphEdge, GraphNode, + Overview, + Page, SearchHit, - SearchResult, - SyncResult, + Stats, + Tag, + Task, Topic, ) __all__ = [ "LibscopeClient", "AsyncLibscopeClient", - "Document", - "SearchResult", - "SearchHit", - "Topic", - "Analytics", "AskResult", + "AskSource", + "Document", + "DocumentView", "Graph", - "GraphNode", "GraphEdge", - "SyncResult", + "GraphNode", + "Overview", + "Page", + "SearchHit", + "Stats", + "Tag", + "Task", + "Topic", "LibscopeError", "LibscopeConnectionError", "NotFoundError", "ValidationError", "ServerError", + "TaskFailedError", ] __version__ = "0.1.0" diff --git a/sdk/python/src/pylibscope/client.py b/sdk/python/src/pylibscope/client.py index cf44401d..c28968cf 100644 --- a/sdk/python/src/pylibscope/client.py +++ b/sdk/python/src/pylibscope/client.py @@ -1,51 +1,100 @@ -"""Synchronous and asynchronous clients for the libscope REST API.""" +"""Synchronous and asynchronous clients for the libscope REST API (``/api/v1``). + +Slow operations (adding documents, syncing connections) run on the server as background +tasks: those methods return a :class:`Task`. Call ``wait_for_task(task.id)`` to wait for it. +""" from __future__ import annotations -from typing import Any, Dict, List, Optional +import asyncio +import time +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional +from urllib.parse import quote import httpx -from pylibscope.connectors import build_connector_config from pylibscope.exceptions import ( LibscopeConnectionError, LibscopeError, NotFoundError, ServerError, + TaskFailedError, ValidationError, ) from pylibscope.models import ( - Analytics, AskResult, Document, + DocumentView, Graph, - GraphEdge, - GraphNode, + Overview, + Page, SearchHit, - SearchResult, - SyncResult, + Tag, + Task, Topic, ) _API = "/api/v1" -_PATH_DOCUMENTS = "/documents" -_PATH_TOPICS = "/topics" +_TAGS = "/tags" +DEFAULT_BASE_URL = "http://localhost:3378" + + +@dataclass(frozen=True) +class _Call: + """One REST request and how to read its ``data``.""" + + method: str + path: str + parse: Callable[[Any], Any] + params: Optional[Dict[str, Any]] = None + json: Optional[Dict[str, Any]] = None + + +def _defined(**values: Any) -> Dict[str, Any]: + """Drop None values (the server applies its defaults).""" + return {k: v for k, v in values.items() if v is not None} + + +def _filters( + topic: Optional[str], + library: Optional[str], + version: Optional[str], + source_type: Optional[str], + tags: Optional[List[str]], +) -> Dict[str, Any]: + return _defined( + topic=topic, library=library, version=version, sourceType=source_type, tags=tags + ) + + +def _query(params: Dict[str, Any]) -> Dict[str, Any]: + """Query-string form: booleans as true/false; lists are sent as repeated keys.""" + return {k: (str(v).lower() if isinstance(v, bool) else v) for k, v in params.items()} + + +def _doc(document_id: str, rest: str = "") -> str: + return f"/documents/{quote(document_id, safe='')}{rest}" + + +def _started(data: Any) -> Task: + return Task(id=data["taskId"], operation=data.get("operation"), status=data["status"]) + + +def _items(model: Any) -> Callable[[Any], Any]: + return lambda data: [model.model_validate(i) for i in data.get("items", [])] def _raise_for_error(response: httpx.Response) -> None: - """Translate HTTP error responses into SDK exceptions.""" + """Translate an error response into an SDK exception.""" if response.status_code < 400: return - try: - body = response.json() - except Exception: - body = {} - - error = body.get("error", {}) + error = response.json().get("error", {}) + except ValueError: + error = {} code = error.get("code", "UNKNOWN") message = error.get("message", response.text) - if response.status_code == 404: raise NotFoundError(message) if response.status_code == 400: @@ -55,70 +104,109 @@ def _raise_for_error(response: httpx.Response) -> None: raise LibscopeError(message, code=code) -def _extract_data(response: httpx.Response) -> Any: - """Extract the ``data`` envelope from a JSON response.""" - body = response.json() - return body.get("data", body) +class _Calls: + """Request builders shared by both clients.""" + @staticmethod + def health() -> _Call: + return _Call("GET", "/health", lambda d: d) -# --------------------------------------------------------------------------- -# Helpers for normalising API responses into SDK models -# --------------------------------------------------------------------------- + @staticmethod + def overview() -> _Call: + return _Call("GET", "/overview", Overview.model_validate) -def _parse_document(raw: Any) -> Document: - if isinstance(raw, dict): - return Document( - id=raw.get("id", ""), - title=raw.get("title", ""), - url=raw.get("url"), - topic=raw.get("topic"), - topic_id=raw.get("topicId"), - tags=raw.get("tags", []), - content_hash=raw.get("contentHash"), - source_type=raw.get("sourceType"), - created_at=raw.get("createdAt"), - updated_at=raw.get("updatedAt"), - ) - return Document.model_validate(raw) - - -def _parse_search_result(data: Any) -> SearchResult: - results = [] - raw_results = data.get("results", []) if isinstance(data, dict) else [] - for r in raw_results: - results.append( - SearchHit( - document_id=r.get("documentId"), - title=r.get("title"), - content=r.get("content"), - score=r.get("score", 0.0), - ) + @staticmethod + def search(query: str, limit: int, offset: int, min_rating: Optional[float], **f: Any) -> _Call: + params = _defined(query=query, limit=limit, offset=offset, minRating=min_rating) + return _Call( + "GET", "/search", Page[SearchHit].model_validate, params=_query({**params, **f}) ) - total = data.get("totalCount", len(results)) if isinstance(data, dict) else len(results) - return SearchResult(results=results, total_count=total) + @staticmethod + def ask(question: str, top_k: Optional[int], min_rating: Optional[float], **f: Any) -> _Call: + body = _defined(question=question, topK=top_k, minRating=min_rating) + return _Call("POST", "/ask", AskResult.model_validate, json={**body, **f}) -def _parse_topic(raw: Any) -> Topic: - if isinstance(raw, dict): - return Topic( - id=raw.get("id", ""), - name=raw.get("name", ""), - parent_id=raw.get("parentId"), - document_count=raw.get("documentCount"), - ) - return Topic.model_validate(raw) + @staticmethod + def add(body: Dict[str, Any]) -> _Call: + return _Call("POST", "/documents", _started, json=body) + @staticmethod + def get_document(document_id: str) -> _Call: + return _Call("GET", _doc(document_id), DocumentView.model_validate) -def _parse_analytics(raw: Any) -> Analytics: - if isinstance(raw, dict): - return Analytics( - total_documents=raw.get("totalDocuments", 0), - total_chunks=raw.get("totalChunks", 0), - total_topics=raw.get("totalTopics", 0), - total_tags=raw.get("totalTags", 0), - database_size_bytes=raw.get("databaseSizeBytes", 0), - ) - return Analytics.model_validate(raw) + @staticmethod + def list_documents(limit: int, offset: int, **f: Any) -> _Call: + params = _query({"limit": limit, "offset": offset, **f}) + return _Call("GET", "/documents", Page[Document].model_validate, params=params) + + @staticmethod + def delete_document(document_id: str) -> _Call: + return _Call("DELETE", _doc(document_id), lambda d: None) + + @staticmethod + def list_topics() -> _Call: + return _Call("GET", "/topics", _items(Topic)) + + @staticmethod + def create_topic(name: str, parent: Optional[str], description: Optional[str]) -> _Call: + body = _defined(name=name, parent=parent, description=description) + return _Call("POST", "/topics", Topic.model_validate, json=body) + + @staticmethod + def add_tags(document_id: str, tags: List[str]) -> _Call: + path = _doc(document_id, _TAGS) + return _Call("POST", path, lambda d: list(d["tags"]), json={"tags": tags}) + + @staticmethod + def remove_tags(document_id: str, tags: List[str]) -> _Call: + path = _doc(document_id, _TAGS) + return _Call("DELETE", path, lambda d: list(d["tags"]), params={"tags": tags}) + + @staticmethod + def list_tags() -> _Call: + return _Call("GET", _TAGS, _items(Tag)) + + @staticmethod + def graph( + topic: Optional[str], tag: Optional[str], threshold: Optional[float], max_nodes: Optional[int] + ) -> _Call: + params = _defined(topic=topic, tag=tag, threshold=threshold, maxNodes=max_nodes) + return _Call("GET", "/graph", Graph.model_validate, params=params) + + @staticmethod + def sync(body: Dict[str, Any]) -> _Call: + return _Call("POST", "/sync", _started, json=body) + + @staticmethod + def get_task(task_id: str) -> _Call: + return _Call("GET", f"/tasks/{quote(task_id, safe='')}", Task.model_validate) + + @staticmethod + def cancel_task(task_id: str) -> _Call: + return _Call("POST", f"/tasks/{quote(task_id, safe='')}/cancel", lambda d: bool(d["cancelRequested"])) + + +def _add_body( + title: Optional[str] = None, + content: Optional[str] = None, + url: Optional[str] = None, + spider: Optional[bool] = None, + max_pages: Optional[int] = None, + **f: Any, +) -> Dict[str, Any]: + body = _defined(title=title, content=content, url=url, spider=spider, maxPages=max_pages) + return {**body, **f} + + +def _finished(task: Task) -> Task: + if task.status != "completed": + raise TaskFailedError(task) + return task + + +def _headers(api_key: Optional[str]) -> Dict[str, str]: + return {"Authorization": f"Bearer {api_key}"} if api_key else {} # --------------------------------------------------------------------------- @@ -127,17 +215,21 @@ def _parse_analytics(raw: Any) -> Analytics: class LibscopeClient: - """Synchronous client for the libscope REST API.""" + """Synchronous client for the libscope REST API. + + ``api_key`` is sent as ``Authorization: Bearer <key>`` (needed when the server sets + ``LIBSCOPE_API_KEY``). + """ def __init__( self, - base_url: str = "http://localhost:3378", + base_url: str = DEFAULT_BASE_URL, timeout: float = 30.0, + api_key: Optional[str] = None, ) -> None: - self._base_url = base_url.rstrip("/") - self._client = httpx.Client(base_url=self._base_url, timeout=timeout) - - # -- context manager ----------------------------------------------------- + self._client = httpx.Client( + base_url=base_url.rstrip("/") + _API, timeout=timeout, headers=_headers(api_key) + ) def __enter__(self) -> "LibscopeClient": return self @@ -149,210 +241,172 @@ def close(self) -> None: """Close the underlying HTTP client.""" self._client.close() - # -- helpers ------------------------------------------------------------- - - def _url(self, path: str) -> str: - return f"{_API}{path}" - - def _request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: + def _run(self, call: _Call) -> Any: try: - resp = self._client.request(method, self._url(path), **kwargs) + resp = self._client.request(call.method, call.path, params=call.params, json=call.json) except httpx.ConnectError as exc: raise LibscopeConnectionError(str(exc)) from exc _raise_for_error(resp) - return resp + return call.parse(resp.json().get("data")) - # -- document operations ------------------------------------------------- + def health(self) -> Dict[str, Any]: + """Liveness check: ``{"status": "ok"}``.""" + return self._run(_Calls.health()) + + def overview(self) -> Overview: + """Counts, topics, installed packs, embedding index and health.""" + return self._run(_Calls.overview()) def search( self, query: str, *, limit: int = 10, + offset: int = 0, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - min_score: Optional[float] = None, - ) -> SearchResult: - """Perform a semantic search across the knowledge base.""" - params: Dict[str, Any] = {"q": query, "limit": limit} - if topic is not None: - params["topic"] = topic - if tags: - params["tag"] = tags[0] - resp = self._request("GET", "/search", params=params) - data = _extract_data(resp) - result = _parse_search_result(data) - if min_score is not None: - result.results = [h for h in result.results if h.score >= min_score] - result.total_count = len(result.results) - return result - - def add_document( + min_rating: Optional[float] = None, + ) -> Page[SearchHit]: + """Search by meaning and keywords.""" + f = _filters(topic, library, version, source_type, tags) + return self._run(_Calls.search(query, limit, offset, min_rating, **f)) + + def ask( self, - url: str, + question: str, *, + top_k: Optional[int] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - ) -> Document: - """Index a document from a URL.""" - payload: Dict[str, Any] = {"url": url} - if topic is not None: - payload["topic"] = topic - if tags: - payload["tags"] = tags - resp = self._request("POST", "/documents/url", json=payload) - return _parse_document(_extract_data(resp)) + min_rating: Optional[float] = None, + ) -> AskResult: + """Answer a question from the knowledge base (needs an LLM on the server).""" + f = _filters(topic, library, version, source_type, tags) + return self._run(_Calls.ask(question, top_k, min_rating, **f)) def add_text( self, title: str, content: str, *, + url: Optional[str] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - ) -> Document: - """Index a document from raw text.""" - payload: Dict[str, Any] = {"title": title, "content": content} - if topic is not None: - payload["topic"] = topic - if tags: - payload["tags"] = tags - resp = self._request("POST", _PATH_DOCUMENTS, json=payload) - return _parse_document(_extract_data(resp)) - - def get_document(self, doc_id: str) -> Document: - """Retrieve a single document by ID.""" - resp = self._request("GET", f"/documents/{doc_id}") - return _parse_document(_extract_data(resp)) + ) -> Task: + """Start adding a document from text. Returns the background task.""" + f = _filters(topic, library, version, source_type, tags) + return self._run(_Calls.add(_add_body(title=title, content=content, url=url, **f))) - def list_documents( + def add_url( self, + url: str, *, + spider: bool = False, + max_pages: Optional[int] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, + tags: Optional[List[str]] = None, + ) -> Task: + """Start adding a web page (``spider=True``: and the pages it links to) or a public + GitHub/GitLab repository. Returns the background task.""" + f = _filters(topic, library, version, source_type, tags) + body = _add_body(url=url, spider=spider or None, max_pages=max_pages, **f) + return self._run(_Calls.add(body)) + + def get_document(self, document_id: str) -> DocumentView: + """Get a document with its content, tags, links and ratings.""" + return self._run(_Calls.get_document(document_id)) + + def list_documents( + self, + *, limit: int = 50, offset: int = 0, - ) -> List[Document]: - """List documents, optionally filtered by topic.""" - params: Dict[str, Any] = {"limit": limit, "offset": offset} - if topic is not None: - params["topic"] = topic - resp = self._request("GET", _PATH_DOCUMENTS, params=params) - data = _extract_data(resp) - if isinstance(data, list): - return [_parse_document(d) for d in data] - return [] - - def delete_document(self, doc_id: str) -> None: - """Delete a document by ID.""" - self._request("DELETE", f"/documents/{doc_id}") - - # -- topic operations ---------------------------------------------------- + topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, + tags: Optional[List[str]] = None, + ) -> Page[Document]: + """List documents, newest first.""" + f = _filters(topic, library, version, source_type, tags) + return self._run(_Calls.list_documents(limit, offset, **f)) + + def delete_document(self, document_id: str) -> None: + """Delete a document.""" + self._run(_Calls.delete_document(document_id)) def list_topics(self) -> List[Topic]: - """List all topics.""" - resp = self._request("GET", _PATH_TOPICS) - data = _extract_data(resp) - if isinstance(data, list): - return [_parse_topic(t) for t in data] - return [] - - def create_topic(self, name: str, *, parent_id: Optional[str] = None) -> Topic: - """Create a new topic.""" - payload: Dict[str, Any] = {"name": name} - if parent_id is not None: - payload["parentId"] = parent_id - resp = self._request("POST", _PATH_TOPICS, json=payload) - return _parse_topic(_extract_data(resp)) - - # -- tag operations ------------------------------------------------------ - - def add_tags(self, doc_id: str, tags: List[str]) -> None: - """Add tags to a document.""" - self._request("POST", f"/documents/{doc_id}/tags", json={"tags": tags}) - - def remove_tags(self, doc_id: str, tags: List[str]) -> None: - """Remove tags from a document. - - Note: The REST API may not support tag removal yet. - This sends a DELETE request to the document tags endpoint. - """ - self._request("DELETE", f"/documents/{doc_id}/tags", json={"tags": tags}) - - def list_tags(self) -> List[str]: - """List all tags in the knowledge base.""" - resp = self._request("GET", "/tags") - data = _extract_data(resp) - if isinstance(data, list): - return [t if isinstance(t, str) else t.get("name", "") for t in data] - return [] - - # -- analytics ----------------------------------------------------------- - - def get_analytics(self) -> Analytics: - """Get knowledge base statistics.""" - resp = self._request("GET", "/stats") - return _parse_analytics(_extract_data(resp)) - - # -- knowledge graph ----------------------------------------------------- - - def get_graph(self, *, min_similarity: float = 0.7) -> Graph: - """Get the knowledge graph. - - Note: This endpoint may not be available in all server versions. - """ - resp = self._request( - "GET", "/graph", params={"min_similarity": min_similarity} - ) - data = _extract_data(resp) - nodes = [GraphNode(**n) for n in data.get("nodes", [])] if isinstance(data, dict) else [] - edges = [GraphEdge(**e) for e in data.get("edges", [])] if isinstance(data, dict) else [] - return Graph(nodes=nodes, edges=edges) + """List topics with their document counts.""" + return self._run(_Calls.list_topics()) - # -- RAG Q&A ------------------------------------------------------------- + def create_topic( + self, name: str, *, parent: Optional[str] = None, description: Optional[str] = None + ) -> Topic: + """Create a topic (``parent``: ID or name of the parent topic).""" + return self._run(_Calls.create_topic(name, parent, description)) - def ask( + def add_tags(self, document_id: str, tags: List[str]) -> List[str]: + """Add tags to a document. Returns the document's tags.""" + return self._run(_Calls.add_tags(document_id, tags)) + + def remove_tags(self, document_id: str, tags: List[str]) -> List[str]: + """Remove tags from a document. Returns the document's remaining tags.""" + return self._run(_Calls.remove_tags(document_id, tags)) + + def list_tags(self) -> List[Tag]: + """List tags with their document counts.""" + return self._run(_Calls.list_tags()) + + def get_graph( self, - question: str, *, topic: Optional[str] = None, - ) -> AskResult: - """Ask a question using RAG-powered Q&A.""" - payload: Dict[str, Any] = {"question": question} - if topic is not None: - payload["topic"] = topic - resp = self._request("POST", "/ask", json=payload) - data = _extract_data(resp) - if isinstance(data, dict): - return AskResult( - answer=data.get("answer", ""), - sources=data.get("sources", []), - ) - return AskResult() - - # -- connector operations ------------------------------------------------ - - def sync_connector(self, connector: str, **config: Any) -> SyncResult: - """Trigger a connector sync. - - Note: This endpoint may not be available in all server versions. - """ - payload = build_connector_config(connector, **config) - resp = self._request("POST", "/connectors/sync", json=payload) - data = _extract_data(resp) - if isinstance(data, dict): - return SyncResult( - connector=data.get("connector", connector), - documents_synced=data.get("documentsSynced", 0), - errors=data.get("errors", []), - ) - return SyncResult(connector=connector) - - # -- health -------------------------------------------------------------- - - def health(self) -> Dict[str, Any]: - """Check server health.""" - resp = self._request("GET", "/health") - return _extract_data(resp) + tag: Optional[str] = None, + threshold: Optional[float] = None, + max_nodes: Optional[int] = None, + ) -> Graph: + """Knowledge graph of documents, topics and tags.""" + return self._run(_Calls.graph(topic, tag, threshold, max_nodes)) + + def sync(self, name: str) -> Task: + """Start syncing a saved connector connection. Returns the background task.""" + return self._run(_Calls.sync({"name": name})) + + def sync_all(self) -> Task: + """Start syncing every saved connection. Returns the background task.""" + return self._run(_Calls.sync({"all": True})) + + def get_task(self, task_id: str) -> Task: + """Status, progress and result of a background task.""" + return self._run(_Calls.get_task(task_id)) + + def cancel_task(self, task_id: str) -> bool: + """Request cancellation. Returns True when the task was still pending or running.""" + return self._run(_Calls.cancel_task(task_id)) + + def wait_for_task(self, task_id: str, *, timeout: float = 300.0, interval: float = 1.0) -> Task: + """Poll a task until it finishes. Raises TaskFailedError when it failed or was + cancelled, and TimeoutError after ``timeout`` seconds.""" + deadline = time.monotonic() + timeout + while True: + task = self.get_task(task_id) + if task.done: + return _finished(task) + if time.monotonic() >= deadline: + raise TimeoutError(f"Task {task_id} did not finish in {timeout} s") + time.sleep(interval) # --------------------------------------------------------------------------- @@ -361,15 +415,17 @@ def health(self) -> Dict[str, Any]: class AsyncLibscopeClient: - """Asynchronous client for the libscope REST API.""" + """Asynchronous client for the libscope REST API. Same methods as LibscopeClient.""" def __init__( self, - base_url: str = "http://localhost:3378", + base_url: str = DEFAULT_BASE_URL, timeout: float = 30.0, + api_key: Optional[str] = None, ) -> None: - self._base_url = base_url.rstrip("/") - self._client = httpx.AsyncClient(base_url=self._base_url, timeout=timeout) + self._client = httpx.AsyncClient( + base_url=base_url.rstrip("/") + _API, timeout=timeout, headers=_headers(api_key) + ) async def __aenter__(self) -> "AsyncLibscopeClient": return self @@ -381,179 +437,149 @@ async def close(self) -> None: """Close the underlying HTTP client.""" await self._client.aclose() - def _url(self, path: str) -> str: - return f"{_API}{path}" - - async def _request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: + async def _run(self, call: _Call) -> Any: try: - resp = await self._client.request(method, self._url(path), **kwargs) + resp = await self._client.request( + call.method, call.path, params=call.params, json=call.json + ) except httpx.ConnectError as exc: raise LibscopeConnectionError(str(exc)) from exc _raise_for_error(resp) - return resp + return call.parse(resp.json().get("data")) + + async def health(self) -> Dict[str, Any]: + return await self._run(_Calls.health()) - # -- document operations ------------------------------------------------- + async def overview(self) -> Overview: + return await self._run(_Calls.overview()) async def search( self, query: str, *, limit: int = 10, + offset: int = 0, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - min_score: Optional[float] = None, - ) -> SearchResult: - params: Dict[str, Any] = {"q": query, "limit": limit} - if topic is not None: - params["topic"] = topic - if tags: - params["tag"] = tags[0] - resp = await self._request("GET", "/search", params=params) - data = _extract_data(resp) - result = _parse_search_result(data) - if min_score is not None: - result.results = [h for h in result.results if h.score >= min_score] - result.total_count = len(result.results) - return result - - async def add_document( + min_rating: Optional[float] = None, + ) -> Page[SearchHit]: + f = _filters(topic, library, version, source_type, tags) + return await self._run(_Calls.search(query, limit, offset, min_rating, **f)) + + async def ask( self, - url: str, + question: str, *, + top_k: Optional[int] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - ) -> Document: - payload: Dict[str, Any] = {"url": url} - if topic is not None: - payload["topic"] = topic - if tags: - payload["tags"] = tags - resp = await self._request("POST", "/documents/url", json=payload) - return _parse_document(_extract_data(resp)) + min_rating: Optional[float] = None, + ) -> AskResult: + f = _filters(topic, library, version, source_type, tags) + return await self._run(_Calls.ask(question, top_k, min_rating, **f)) async def add_text( self, title: str, content: str, *, + url: Optional[str] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, tags: Optional[List[str]] = None, - ) -> Document: - payload: Dict[str, Any] = {"title": title, "content": content} - if topic is not None: - payload["topic"] = topic - if tags: - payload["tags"] = tags - resp = await self._request("POST", _PATH_DOCUMENTS, json=payload) - return _parse_document(_extract_data(resp)) - - async def get_document(self, doc_id: str) -> Document: - resp = await self._request("GET", f"/documents/{doc_id}") - return _parse_document(_extract_data(resp)) + ) -> Task: + f = _filters(topic, library, version, source_type, tags) + return await self._run(_Calls.add(_add_body(title=title, content=content, url=url, **f))) - async def list_documents( + async def add_url( self, + url: str, *, + spider: bool = False, + max_pages: Optional[int] = None, topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, + tags: Optional[List[str]] = None, + ) -> Task: + f = _filters(topic, library, version, source_type, tags) + body = _add_body(url=url, spider=spider or None, max_pages=max_pages, **f) + return await self._run(_Calls.add(body)) + + async def get_document(self, document_id: str) -> DocumentView: + return await self._run(_Calls.get_document(document_id)) + + async def list_documents( + self, + *, limit: int = 50, offset: int = 0, - ) -> List[Document]: - params: Dict[str, Any] = {"limit": limit, "offset": offset} - if topic is not None: - params["topic"] = topic - resp = await self._request("GET", _PATH_DOCUMENTS, params=params) - data = _extract_data(resp) - if isinstance(data, list): - return [_parse_document(d) for d in data] - return [] - - async def delete_document(self, doc_id: str) -> None: - await self._request("DELETE", f"/documents/{doc_id}") + topic: Optional[str] = None, + library: Optional[str] = None, + version: Optional[str] = None, + source_type: Optional[str] = None, + tags: Optional[List[str]] = None, + ) -> Page[Document]: + f = _filters(topic, library, version, source_type, tags) + return await self._run(_Calls.list_documents(limit, offset, **f)) - # -- topic operations ---------------------------------------------------- + async def delete_document(self, document_id: str) -> None: + await self._run(_Calls.delete_document(document_id)) async def list_topics(self) -> List[Topic]: - resp = await self._request("GET", _PATH_TOPICS) - data = _extract_data(resp) - if isinstance(data, list): - return [_parse_topic(t) for t in data] - return [] - - async def create_topic(self, name: str, *, parent_id: Optional[str] = None) -> Topic: - payload: Dict[str, Any] = {"name": name} - if parent_id is not None: - payload["parentId"] = parent_id - resp = await self._request("POST", _PATH_TOPICS, json=payload) - return _parse_topic(_extract_data(resp)) + return await self._run(_Calls.list_topics()) - # -- tag operations ------------------------------------------------------ + async def create_topic( + self, name: str, *, parent: Optional[str] = None, description: Optional[str] = None + ) -> Topic: + return await self._run(_Calls.create_topic(name, parent, description)) - async def add_tags(self, doc_id: str, tags: List[str]) -> None: - await self._request("POST", f"/documents/{doc_id}/tags", json={"tags": tags}) + async def add_tags(self, document_id: str, tags: List[str]) -> List[str]: + return await self._run(_Calls.add_tags(document_id, tags)) - async def remove_tags(self, doc_id: str, tags: List[str]) -> None: - await self._request("DELETE", f"/documents/{doc_id}/tags", json={"tags": tags}) + async def remove_tags(self, document_id: str, tags: List[str]) -> List[str]: + return await self._run(_Calls.remove_tags(document_id, tags)) - async def list_tags(self) -> List[str]: - resp = await self._request("GET", "/tags") - data = _extract_data(resp) - if isinstance(data, list): - return [t if isinstance(t, str) else t.get("name", "") for t in data] - return [] + async def list_tags(self) -> List[Tag]: + return await self._run(_Calls.list_tags()) - # -- analytics ----------------------------------------------------------- - - async def get_analytics(self) -> Analytics: - resp = await self._request("GET", "/stats") - return _parse_analytics(_extract_data(resp)) - - # -- knowledge graph ----------------------------------------------------- - - async def get_graph(self, *, min_similarity: float = 0.7) -> Graph: - resp = await self._request( - "GET", "/graph", params={"min_similarity": min_similarity} - ) - data = _extract_data(resp) - nodes = [GraphNode(**n) for n in data.get("nodes", [])] if isinstance(data, dict) else [] - edges = [GraphEdge(**e) for e in data.get("edges", [])] if isinstance(data, dict) else [] - return Graph(nodes=nodes, edges=edges) - - # -- RAG Q&A ------------------------------------------------------------- - - async def ask( + async def get_graph( self, - question: str, *, topic: Optional[str] = None, - ) -> AskResult: - payload: Dict[str, Any] = {"question": question} - if topic is not None: - payload["topic"] = topic - resp = await self._request("POST", "/ask", json=payload) - data = _extract_data(resp) - if isinstance(data, dict): - return AskResult( - answer=data.get("answer", ""), - sources=data.get("sources", []), - ) - return AskResult() - - # -- connector operations ------------------------------------------------ - - async def sync_connector(self, connector: str, **config: Any) -> SyncResult: - payload = build_connector_config(connector, **config) - resp = await self._request("POST", "/connectors/sync", json=payload) - data = _extract_data(resp) - if isinstance(data, dict): - return SyncResult( - connector=data.get("connector", connector), - documents_synced=data.get("documentsSynced", 0), - errors=data.get("errors", []), - ) - return SyncResult(connector=connector) - - # -- health -------------------------------------------------------------- - - async def health(self) -> Dict[str, Any]: - resp = await self._request("GET", "/health") - return _extract_data(resp) + tag: Optional[str] = None, + threshold: Optional[float] = None, + max_nodes: Optional[int] = None, + ) -> Graph: + return await self._run(_Calls.graph(topic, tag, threshold, max_nodes)) + + async def sync(self, name: str) -> Task: + return await self._run(_Calls.sync({"name": name})) + + async def sync_all(self) -> Task: + return await self._run(_Calls.sync({"all": True})) + + async def get_task(self, task_id: str) -> Task: + return await self._run(_Calls.get_task(task_id)) + + async def cancel_task(self, task_id: str) -> bool: + return await self._run(_Calls.cancel_task(task_id)) + + async def wait_for_task(self, task_id: str, *, interval: float = 1.0) -> Task: + """Poll a task until it finishes. Raises TaskFailedError when it failed or was + cancelled. It has no time limit: set one with ``asyncio.wait_for(..., timeout)``, or + ``async with asyncio.timeout(...)`` on Python 3.11+.""" + while True: + task = await self.get_task(task_id) + if task.done: + return _finished(task) + await asyncio.sleep(interval) diff --git a/sdk/python/src/pylibscope/connectors.py b/sdk/python/src/pylibscope/connectors.py deleted file mode 100644 index 14e2c909..00000000 --- a/sdk/python/src/pylibscope/connectors.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Connector management helpers for libscope SDK.""" - -from __future__ import annotations - -from typing import Any, Dict - - -def build_connector_config(connector: str, **kwargs: Any) -> Dict[str, Any]: - """Build a connector configuration payload. - - Args: - connector: Connector name (e.g., "obsidian", "onenote"). - **kwargs: Connector-specific configuration options. - - Returns: - A dict suitable for posting to the connector sync endpoint. - """ - return {"connector": connector, "config": kwargs} diff --git a/sdk/python/src/pylibscope/exceptions.py b/sdk/python/src/pylibscope/exceptions.py index c9368b2e..25cdd26d 100644 --- a/sdk/python/src/pylibscope/exceptions.py +++ b/sdk/python/src/pylibscope/exceptions.py @@ -1,5 +1,12 @@ """Custom exceptions for the libscope Python SDK.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pylibscope.models import Task + class LibscopeError(Exception): """Base exception for all libscope SDK errors.""" @@ -35,3 +42,12 @@ class ServerError(LibscopeError): def __init__(self, message: str = "Internal server error") -> None: super().__init__(message, code="INTERNAL_ERROR") + + +class TaskFailedError(LibscopeError): + """Raised by ``wait_for_task`` when a background task failed or was cancelled.""" + + def __init__(self, task: Task) -> None: + self.task = task + reason = task.error or task.status + super().__init__(f"Task {task.id} ({task.operation}) {reason}", code="TASK_FAILED") diff --git a/sdk/python/src/pylibscope/models.py b/sdk/python/src/pylibscope/models.py index 9f128b2a..aee2979a 100644 --- a/sdk/python/src/pylibscope/models.py +++ b/sdk/python/src/pylibscope/models.py @@ -1,95 +1,180 @@ -"""Pydantic models for libscope API types.""" +"""Pydantic models for libscope REST API results. + +Field names are snake_case; responses (camelCase) are parsed with aliases. +""" from __future__ import annotations -from typing import Any, List, Optional +import json +from typing import Any, Dict, Generic, List, Optional, TypeVar -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, field_validator +from pydantic.alias_generators import to_camel +T = TypeVar("T") -class Document(BaseModel): - """A document stored in the knowledge base.""" - id: str +class _Model(BaseModel): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + +class Page(_Model, Generic[T]): + """One page of a list result.""" + + items: List[T] = [] + total: int = 0 + limit: int = 0 + offset: int = 0 + + +class Document(_Model): + """A document without its content (list results).""" + + document_id: str title: str - url: Optional[str] = None - topic: Optional[str] = None + source_type: Optional[str] = None + library: Optional[str] = None + version: Optional[str] = None topic_id: Optional[str] = None - tags: List[str] = [] + url: Optional[str] = None content_hash: Optional[str] = None - source_type: Optional[str] = None + submitted_by: Optional[str] = None + content_length: Optional[int] = None created_at: Optional[str] = None updated_at: Optional[str] = None -class SearchHit(BaseModel): - """A single search result with relevance score.""" +class DocumentView(_Model): + """A document with (a page of) its content, tags, links and rating summary.""" - document_id: Optional[str] = None - title: Optional[str] = None - content: Optional[str] = None - score: float = 0.0 + document: Document + content: str = "" + content_length: int = 0 + offset: int = 0 + next_offset: Optional[int] = None + tags: List[str] = [] + links: Dict[str, Any] = {} + ratings: Dict[str, Any] = {} -class SearchResult(BaseModel): - """Response from a search query.""" +class SearchHit(_Model): + """A matching chunk.""" - results: List[SearchHit] = [] - total_count: int = 0 + document_id: str + chunk_id: str + title: str + content: str = "" + source_type: Optional[str] = None + library: Optional[str] = None + version: Optional[str] = None + topic_id: Optional[str] = None + url: Optional[str] = None + score: float = 0.0 + avg_rating: Optional[float] = None -class Topic(BaseModel): - """A topic/category for organizing documents.""" +class Topic(_Model): + """A topic for organizing documents.""" id: str name: str + description: Optional[str] = None parent_id: Optional[str] = None document_count: Optional[int] = None -class Analytics(BaseModel): - """Knowledge base statistics.""" +class Tag(_Model): + """A tag with its document count.""" + + id: str + name: str + document_count: int = 0 + + +class Stats(_Model): + """Counts in an overview.""" total_documents: int = 0 total_chunks: int = 0 total_topics: int = 0 - total_tags: int = 0 database_size_bytes: int = 0 -class AskResult(BaseModel): - """Response from a RAG question-answering query.""" +class Overview(_Model): + """Counts, topics, installed packs, embedding index and health.""" + + stats: Stats = Stats() + topics: List[Topic] = [] + packs: List[Dict[str, Any]] = [] + index: Dict[str, Any] = {} + health: Dict[str, Any] = {} + - answer: str = "" - sources: List[Any] = [] +class AskSource(_Model): + """A chunk an answer is based on.""" + document_id: str + title: str + chunk: str = "" + score: float = 0.0 + + +class AskResult(_Model): + """An answer (mode "answer"), or the context to answer from (mode "context").""" + + mode: str = "answer" + answer: Optional[str] = None + context_prompt: Optional[str] = None + sources: List[AskSource] = [] + model: Optional[str] = None + tokens_used: Optional[int] = None -class GraphNode(BaseModel): - """A node in the knowledge graph.""" + +class GraphNode(_Model): + """A node in the knowledge graph (document, topic or tag).""" id: str label: str type: Optional[str] = None + metadata: Dict[str, Any] = {} -class GraphEdge(BaseModel): +class GraphEdge(_Model): """An edge in the knowledge graph.""" source: str target: str + type: Optional[str] = None weight: float = 0.0 -class Graph(BaseModel): - """Knowledge graph representation.""" +class Graph(_Model): + """Knowledge graph.""" nodes: List[GraphNode] = [] edges: List[GraphEdge] = [] -class SyncResult(BaseModel): - """Result from a connector sync operation.""" +class Task(_Model): + """A background task. ``result`` is the operation result once completed.""" - connector: str - documents_synced: int = 0 - errors: List[str] = [] + id: str + operation: Optional[str] = None + status: str + progress: Optional[Dict[str, Any]] = None + result: Any = None + error: Optional[str] = None + created_at: Optional[str] = None + started_at: Optional[str] = None + completed_at: Optional[str] = None + + @field_validator("result", mode="before") + @classmethod + def _decode_result(cls, value: Any) -> Any: + # The server sends the operation result as a JSON string. + return json.loads(value) if isinstance(value, str) else value + + @property + def done(self) -> bool: + """True when the task completed, failed or was cancelled.""" + return self.status in ("completed", "failed", "cancelled") diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 9ca5002d..ceb4599c 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -1,4 +1,7 @@ -"""Tests for libscope sync and async clients.""" +"""Tests for the libscope sync and async clients (HTTP mocked with respx).""" + +import asyncio +import json import httpx import pytest @@ -7,368 +10,316 @@ from pylibscope.client import AsyncLibscopeClient, LibscopeClient from pylibscope.exceptions import ( LibscopeConnectionError, + LibscopeError, NotFoundError, ServerError, + TaskFailedError, ValidationError, ) -from pylibscope.models import Analytics, AskResult, Document, SearchResult, Topic BASE = "http://localhost:3378" API = f"{BASE}/api/v1" +HIT = { + "documentId": "d1", + "chunkId": "c1", + "title": "Doc 1", + "content": "hello", + "sourceType": "manual", + "score": 0.9, +} +DOC = {"documentId": "d1", "title": "Doc 1", "sourceType": "manual", "contentLength": 5} +STARTED = {"taskId": "t1", "operation": "add", "status": "running"} +ADD_RESULT = {"kind": "content", "documents": [{"documentId": "d1", "title": "Doc 1"}]} -# --------------------------------------------------------------------------- -# Synchronous client tests -# --------------------------------------------------------------------------- +def ok(data, status=200): + return httpx.Response(status, json={"data": data, "meta": {"took": 1}}) + + +def task(status, **extra): + return {"id": "t1", "operation": "add", "status": status, **extra} + + +def body(route): + return json.loads(route.calls[0].request.content) -class TestLibscopeClientSearch: - @respx.mock - def test_search(self): - respx.get(f"{API}/search").mock( - return_value=httpx.Response( - 200, - json={ - "data": { - "results": [ - {"documentId": "d1", "title": "Doc 1", "content": "hello", "score": 0.9} - ], - "totalCount": 1, - }, - "meta": {"took": 10}, - }, - ) - ) - with LibscopeClient() as client: - result = client.search("hello") - assert isinstance(result, SearchResult) - assert result.total_count == 1 - assert result.results[0].score == 0.9 +class TestSearch: @respx.mock - def test_search_with_filters(self): + def test_search_sends_operation_names_and_parses_page(self): route = respx.get(f"{API}/search").mock( - return_value=httpx.Response(200, json={"data": {"results": [], "totalCount": 0}}) + return_value=ok({"items": [HIT], "total": 1, "limit": 5, "offset": 0}) ) with LibscopeClient() as client: - client.search("test", topic="python", tags=["tutorial"], limit=5) - assert "topic=python" in str(route.calls[0].request.url) - assert "tag=tutorial" in str(route.calls[0].request.url) - assert "limit=5" in str(route.calls[0].request.url) - - @respx.mock - def test_search_min_score_filter(self): - respx.get(f"{API}/search").mock( - return_value=httpx.Response( - 200, - json={ - "data": { - "results": [ - {"documentId": "d1", "title": "High", "content": "", "score": 0.95}, - {"documentId": "d2", "title": "Low", "content": "", "score": 0.3}, - ], - "totalCount": 2, - } - }, + page = client.search( + "hello", limit=5, topic="py", source_type="library", tags=["a", "b"], min_rating=3 ) - ) - with LibscopeClient() as client: - result = client.search("test", min_score=0.5) - assert len(result.results) == 1 - assert result.results[0].document_id == "d1" - - -class TestLibscopeClientDocuments: + params = route.calls[0].request.url.params + assert params["query"] == "hello" + assert params["limit"] == "5" + assert params["sourceType"] == "library" + assert params.get_list("tags") == ["a", "b"] + assert params["minRating"] == "3" + assert page.total == 1 + assert page.items[0].document_id == "d1" + assert page.items[0].chunk_id == "c1" + + +class TestDocuments: @respx.mock - def test_add_document_url(self): - respx.post(f"{API}/documents/url").mock( - return_value=httpx.Response( - 201, - json={"data": {"id": "d1", "title": "Python Tutorial", "url": "https://example.com"}}, - ) - ) + def test_add_text_starts_a_task(self): + route = respx.post(f"{API}/documents").mock(return_value=ok(STARTED, 202)) with LibscopeClient() as client: - doc = client.add_document("https://example.com") - assert isinstance(doc, Document) - assert doc.id == "d1" + started = client.add_text("T", "C", topic="py", tags=["x"]) + assert body(route) == {"title": "T", "content": "C", "topic": "py", "tags": ["x"]} + assert (started.id, started.status, started.operation) == ("t1", "running", "add") @respx.mock - def test_add_text(self): - respx.post(f"{API}/documents").mock( - return_value=httpx.Response( - 201, json={"data": {"id": "d2", "title": "My Doc"}} - ) - ) + def test_add_url_with_spider(self): + route = respx.post(f"{API}/documents").mock(return_value=ok(STARTED, 202)) with LibscopeClient() as client: - doc = client.add_text("My Doc", "Some content") - assert doc.title == "My Doc" + client.add_url("https://example.com", spider=True, max_pages=5) + client.add_url("https://example.com/page") + assert body(route) == {"url": "https://example.com", "spider": True, "maxPages": 5} + assert json.loads(route.calls[1].request.content) == {"url": "https://example.com/page"} @respx.mock def test_get_document(self): - respx.get(f"{API}/documents/d1").mock( - return_value=httpx.Response(200, json={"data": {"id": "d1", "title": "Test"}}) + respx.get(f"{API}/documents/d%2F1").mock( + return_value=ok({"document": DOC, "content": "hello", "tags": ["a"], "nextOffset": None}) ) with LibscopeClient() as client: - doc = client.get_document("d1") - assert doc.id == "d1" + view = client.get_document("d/1") + assert view.document.title == "Doc 1" + assert view.content == "hello" + assert view.tags == ["a"] @respx.mock def test_list_documents(self): - respx.get(f"{API}/documents").mock( - return_value=httpx.Response( - 200, - json={"data": [{"id": "d1", "title": "A"}, {"id": "d2", "title": "B"}]}, - ) + route = respx.get(f"{API}/documents").mock( + return_value=ok({"items": [DOC], "total": 1, "limit": 10, "offset": 0}) ) with LibscopeClient() as client: - docs = client.list_documents() - assert len(docs) == 2 + page = client.list_documents(limit=10, library="react") + assert route.calls[0].request.url.params["library"] == "react" + assert page.items[0].content_length == 5 @respx.mock def test_delete_document(self): - respx.delete(f"{API}/documents/d1").mock( - return_value=httpx.Response(200, json={"data": {"deleted": True}}) + route = respx.delete(f"{API}/documents/d1").mock( + return_value=ok({"documentId": "d1", "deleted": True}) ) with LibscopeClient() as client: - client.delete_document("d1") # should not raise + assert client.delete_document("d1") is None + assert route.called -class TestLibscopeClientTopics: +class TestTopicsTagsGraph: @respx.mock - def test_list_topics(self): + def test_topics(self): respx.get(f"{API}/topics").mock( - return_value=httpx.Response( - 200, - json={"data": [{"id": "t1", "name": "Python"}, {"id": "t2", "name": "Rust"}]}, - ) + return_value=ok({"items": [{"id": "py", "name": "Python", "documentCount": 2}]}) ) + create = respx.post(f"{API}/topics").mock(return_value=ok({"id": "web", "name": "Web"})) with LibscopeClient() as client: topics = client.list_topics() - assert len(topics) == 2 - assert all(isinstance(t, Topic) for t in topics) + created = client.create_topic("Web", parent="py") + assert topics[0].document_count == 2 + assert body(create) == {"name": "Web", "parent": "py"} + assert created.id == "web" @respx.mock - def test_create_topic(self): - respx.post(f"{API}/topics").mock( - return_value=httpx.Response(201, json={"data": {"id": "t3", "name": "Go"}}) + def test_tags(self): + add = respx.post(f"{API}/documents/d1/tags").mock( + return_value=ok({"documentId": "d1", "tags": ["a", "b"]}) ) - with LibscopeClient() as client: - topic = client.create_topic("Go") - assert topic.name == "Go" - - -class TestLibscopeClientTags: - @respx.mock - def test_add_tags(self): - respx.post(f"{API}/documents/d1/tags").mock( - return_value=httpx.Response(200, json={"data": ["alpha", "beta"]}) + remove = respx.delete(f"{API}/documents/d1/tags").mock( + return_value=ok({"documentId": "d1", "removed": ["b"], "tags": ["a"]}) + ) + respx.get(f"{API}/tags").mock( + return_value=ok({"items": [{"id": "1", "name": "a", "documentCount": 1}]}) ) with LibscopeClient() as client: - client.add_tags("d1", ["alpha", "beta"]) + assert client.add_tags("d1", ["b"]) == ["a", "b"] + assert client.remove_tags("d1", ["b"]) == ["a"] + assert client.list_tags()[0].name == "a" + assert body(add) == {"tags": ["b"]} + assert remove.calls[0].request.url.params.get_list("tags") == ["b"] @respx.mock - def test_list_tags(self): - respx.get(f"{API}/tags").mock( - return_value=httpx.Response(200, json={"data": ["python", "tutorial"]}) + def test_graph(self): + route = respx.get(f"{API}/graph").mock( + return_value=ok( + { + "nodes": [{"id": "d1", "label": "Doc", "type": "document"}], + "edges": [{"source": "d1", "target": "t", "type": "has_tag", "weight": 1}], + } + ) ) with LibscopeClient() as client: - tags = client.list_tags() - assert tags == ["python", "tutorial"] + graph = client.get_graph(threshold=0.5) + assert route.calls[0].request.url.params["threshold"] == "0.5" + assert graph.nodes[0].type == "document" + assert graph.edges[0].weight == 1 -class TestLibscopeClientAnalytics: +class TestOverviewAsk: @respx.mock - def test_get_analytics(self): - respx.get(f"{API}/stats").mock( - return_value=httpx.Response( - 200, - json={ - "data": { - "totalDocuments": 42, - "totalChunks": 100, - "totalTopics": 3, - "totalTags": 7, - "databaseSizeBytes": 2048, - } - }, + def test_overview(self): + respx.get(f"{API}/overview").mock( + return_value=ok( + { + "stats": {"totalDocuments": 3, "totalChunks": 9, "databaseSizeBytes": 4096}, + "topics": [], + "packs": [], + "health": {"database": "ok"}, + } ) ) with LibscopeClient() as client: - stats = client.get_analytics() - assert isinstance(stats, Analytics) - assert stats.total_documents == 42 - + overview = client.overview() + assert overview.stats.total_documents == 3 + assert overview.health["database"] == "ok" -class TestLibscopeClientAsk: @respx.mock def test_ask(self): - respx.post(f"{API}/ask").mock( - return_value=httpx.Response( - 200, - json={"data": {"answer": "Use decorators.", "sources": [{"id": "d1"}]}}, + route = respx.post(f"{API}/ask").mock( + return_value=ok( + { + "mode": "answer", + "answer": "42", + "model": "m", + "sources": [{"documentId": "d1", "title": "Doc", "chunk": "x", "score": 0.5}], + } ) ) with LibscopeClient() as client: - result = client.ask("How to use decorators?") - assert isinstance(result, AskResult) - assert result.answer == "Use decorators." + result = client.ask("why?", top_k=3, source_type="manual") + assert body(route) == {"question": "why?", "topK": 3, "sourceType": "manual"} + assert result.answer == "42" + assert result.sources[0].document_id == "d1" - -class TestLibscopeClientHealth: @respx.mock def test_health(self): - respx.get(f"{API}/health").mock( - return_value=httpx.Response( - 200, json={"data": {"status": "ok", "docCount": 10}} - ) - ) + respx.get(f"{API}/health").mock(return_value=ok({"status": "ok"})) with LibscopeClient() as client: - h = client.health() - assert h["status"] == "ok" + assert client.health() == {"status": "ok"} -class TestLibscopeClientErrors: +class TestTasks: @respx.mock - def test_not_found(self): - respx.get(f"{API}/documents/missing").mock( - return_value=httpx.Response( - 404, json={"error": {"code": "NOT_FOUND", "message": "Document not found"}} - ) + def test_wait_for_task_returns_the_parsed_result(self): + respx.get(f"{API}/tasks/t1").mock( + side_effect=[ + ok(task("running", progress={"current": 1, "total": 2})), + ok(task("completed", result=json.dumps(ADD_RESULT))), + ] ) with LibscopeClient() as client: - with pytest.raises(NotFoundError): - client.get_document("missing") + done = client.wait_for_task("t1", interval=0) + assert done.status == "completed" + assert done.result["documents"][0]["documentId"] == "d1" @respx.mock - def test_validation_error(self): - respx.post(f"{API}/documents").mock( - return_value=httpx.Response( - 400, - json={"error": {"code": "VALIDATION_ERROR", "message": "title is required"}}, - ) - ) + def test_wait_for_task_raises_when_the_task_failed(self): + respx.get(f"{API}/tasks/t1").mock(return_value=ok(task("failed", error="boom"))) with LibscopeClient() as client: - with pytest.raises(ValidationError): - client.add_text("", "") + with pytest.raises(TaskFailedError, match="boom") as exc: + client.wait_for_task("t1", interval=0) + assert exc.value.task.status == "failed" @respx.mock - def test_server_error(self): - respx.get(f"{API}/stats").mock( - return_value=httpx.Response( - 500, json={"error": {"code": "INTERNAL_ERROR", "message": "oops"}} - ) - ) + def test_wait_for_task_times_out(self): + respx.get(f"{API}/tasks/t1").mock(return_value=ok(task("running"))) with LibscopeClient() as client: - with pytest.raises(ServerError): - client.get_analytics() + with pytest.raises(TimeoutError): + client.wait_for_task("t1", timeout=0, interval=0) - def test_connection_refused(self): - client = LibscopeClient(base_url="http://localhost:19999") - with pytest.raises(LibscopeConnectionError): - client.search("hello") - client.close() - - -class TestLibscopeClientContextManager: @respx.mock - def test_context_manager(self): - respx.get(f"{API}/health").mock( - return_value=httpx.Response(200, json={"data": {"status": "ok"}}) + def test_sync_and_cancel(self): + sync = respx.post(f"{API}/sync").mock(return_value=ok({**STARTED, "operation": "sync"}, 202)) + respx.post(f"{API}/tasks/t1/cancel").mock( + return_value=ok({"taskId": "t1", "cancelRequested": True, "status": "cancelled"}) ) with LibscopeClient() as client: - h = client.health() - assert h["status"] == "ok" - - -# --------------------------------------------------------------------------- -# Async client tests -# --------------------------------------------------------------------------- - - -class TestAsyncLibscopeClient: - @pytest.mark.asyncio + assert client.sync("notes").operation == "sync" + client.sync_all() + assert client.cancel_task("t1") is True + assert body(sync) == {"name": "notes"} + assert json.loads(sync.calls[1].request.content) == {"all": True} + + +class TestErrors: + @pytest.mark.parametrize( + "status,error", + [(404, NotFoundError), (400, ValidationError), (500, ServerError)], + ) @respx.mock - async def test_search(self): - respx.get(f"{API}/search").mock( - return_value=httpx.Response( - 200, - json={ - "data": { - "results": [ - {"documentId": "d1", "title": "Doc", "content": "hi", "score": 0.8} - ], - "totalCount": 1, - } - }, - ) + def test_status_codes(self, status, error): + respx.get(f"{API}/documents/x").mock( + return_value=httpx.Response(status, json={"error": {"code": "C", "message": "msg"}}) ) - async with AsyncLibscopeClient() as client: - result = await client.search("hi") - assert result.total_count == 1 + with LibscopeClient() as client: + with pytest.raises(error, match="msg"): + client.get_document("x") - @pytest.mark.asyncio @respx.mock - async def test_add_text(self): - respx.post(f"{API}/documents").mock( - return_value=httpx.Response(201, json={"data": {"id": "d1", "title": "Test"}}) + def test_other_status_keeps_the_server_code(self): + respx.get(f"{API}/health").mock( + return_value=httpx.Response(401, json={"error": {"code": "UNAUTHORIZED", "message": "no"}}) ) - async with AsyncLibscopeClient() as client: - doc = await client.add_text("Test", "content") - assert doc.id == "d1" + with LibscopeClient() as client: + with pytest.raises(LibscopeError) as exc: + client.health() + assert exc.value.code == "UNAUTHORIZED" - @pytest.mark.asyncio @respx.mock - async def test_list_documents(self): - respx.get(f"{API}/documents").mock( - return_value=httpx.Response( - 200, json={"data": [{"id": "d1", "title": "A"}]} - ) - ) - async with AsyncLibscopeClient() as client: - docs = await client.list_documents() - assert len(docs) == 1 + def test_connection_refused(self): + respx.get(f"{API}/health").mock(side_effect=httpx.ConnectError("refused")) + with LibscopeClient() as client: + with pytest.raises(LibscopeConnectionError): + client.health() - @pytest.mark.asyncio @respx.mock - async def test_error_handling(self): - respx.get(f"{API}/documents/bad").mock( - return_value=httpx.Response( - 404, json={"error": {"code": "NOT_FOUND", "message": "not found"}} - ) - ) - async with AsyncLibscopeClient() as client: - with pytest.raises(NotFoundError): - await client.get_document("bad") + def test_api_key_is_sent_as_bearer_token(self): + route = respx.get("http://server:9000/api/v1/health").mock(return_value=ok({"status": "ok"})) + with LibscopeClient("http://server:9000/", api_key="secret") as client: + client.health() + assert route.calls[0].request.headers["Authorization"] == "Bearer secret" + +class TestAsyncClient: @pytest.mark.asyncio @respx.mock - async def test_get_analytics(self): - respx.get(f"{API}/stats").mock( - return_value=httpx.Response( - 200, - json={"data": {"totalDocuments": 5, "totalChunks": 20, "totalTopics": 1, "totalTags": 3, "databaseSizeBytes": 512}}, - ) + async def test_add_wait_search(self): + respx.post(f"{API}/documents").mock(return_value=ok(STARTED, 202)) + respx.get(f"{API}/tasks/t1").mock( + return_value=ok(task("completed", result=json.dumps(ADD_RESULT))) ) + respx.get(f"{API}/search").mock(return_value=ok({"items": [HIT], "total": 1})) async with AsyncLibscopeClient() as client: - stats = await client.get_analytics() - assert stats.total_documents == 5 + started = await client.add_text("T", "C") + done = await client.wait_for_task(started.id, interval=0) + page = await client.search("hello") + assert done.result["kind"] == "content" + assert page.items[0].title == "Doc 1" @pytest.mark.asyncio @respx.mock - async def test_ask(self): - respx.post(f"{API}/ask").mock( - return_value=httpx.Response(200, json={"data": {"answer": "yes", "sources": []}}) - ) + async def test_wait_for_task_is_bounded_by_the_caller(self): + respx.get(f"{API}/tasks/t1").mock(return_value=ok(task("running"))) async with AsyncLibscopeClient() as client: - result = await client.ask("is it?") - assert result.answer == "yes" + waiting = client.wait_for_task("t1", interval=0) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(waiting, timeout=0.05) @pytest.mark.asyncio @respx.mock - async def test_context_manager(self): - respx.get(f"{API}/health").mock( - return_value=httpx.Response(200, json={"data": {"status": "ok"}}) + async def test_errors(self): + respx.get(f"{API}/documents/x").mock( + return_value=httpx.Response(404, json={"error": {"code": "N", "message": "gone"}}) ) async with AsyncLibscopeClient() as client: - h = await client.health() - assert h["status"] == "ok" + with pytest.raises(NotFoundError): + await client.get_document("x") diff --git a/sdk/python/tests/test_models.py b/sdk/python/tests/test_models.py index 1f472bcd..89344979 100644 --- a/sdk/python/tests/test_models.py +++ b/sdk/python/tests/test_models.py @@ -1,105 +1,48 @@ """Tests for libscope Pydantic models.""" -from pylibscope.models import ( - Analytics, - AskResult, - Document, - Graph, - GraphEdge, - GraphNode, - SearchHit, - SearchResult, - SyncResult, - Topic, -) +import json +from pylibscope.models import AskResult, Document, Page, SearchHit, Task, Topic -class TestDocument: - def test_minimal(self): - doc = Document(id="1", title="Hello") - assert doc.id == "1" - assert doc.title == "Hello" - assert doc.url is None - assert doc.tags == [] - def test_full(self): - doc = Document( - id="abc", - title="Test", - url="https://example.com", - topic="python", - topic_id="t1", - tags=["a", "b"], - content_hash="sha256:...", - source_type="manual", - created_at="2024-01-01", - updated_at="2024-01-02", - ) - assert doc.url == "https://example.com" - assert doc.tags == ["a", "b"] - assert doc.source_type == "manual" +def test_models_read_camel_case_and_accept_snake_case(): + doc = Document.model_validate({"documentId": "d1", "title": "T", "sourceType": "manual"}) + assert (doc.document_id, doc.source_type) == ("d1", "manual") + assert Document(document_id="d2", title="U").document_id == "d2" -class TestSearchResult: - def test_empty(self): - sr = SearchResult() - assert sr.results == [] - assert sr.total_count == 0 +def test_page_of_hits(): + page = Page[SearchHit].model_validate( + { + "items": [{"documentId": "d", "chunkId": "c", "title": "T", "score": 0.4}], + "total": 7, + "limit": 1, + "offset": 2, + } + ) + assert page.items[0].score == 0.4 + assert (page.total, page.limit, page.offset) == (7, 1, 2) - def test_with_hits(self): - hit = SearchHit(document_id="1", title="Doc", content="text", score=0.95) - sr = SearchResult(results=[hit], total_count=1) - assert len(sr.results) == 1 - assert sr.results[0].score == 0.95 - assert sr.results[0].document_id == "1" +def test_task_decodes_its_json_result(): + task = Task.model_validate( + {"id": "t", "status": "completed", "result": json.dumps({"items": [1]})} + ) + assert task.result == {"items": [1]} + assert task.done + assert not Task(id="t", status="running").done -class TestTopic: - def test_basic(self): - t = Topic(id="t1", name="Python") - assert t.name == "Python" - assert t.parent_id is None - def test_with_parent(self): - t = Topic(id="t2", name="Flask", parent_id="t1") - assert t.parent_id == "t1" +def test_ask_result_in_context_mode(): + result = AskResult.model_validate( + {"mode": "context", "contextPrompt": "ctx", "sources": [], "unknown": 1} + ) + assert result.mode == "context" + assert result.context_prompt == "ctx" + assert result.answer is None -class TestAnalytics: - def test_defaults(self): - a = Analytics() - assert a.total_documents == 0 - assert a.database_size_bytes == 0 - - def test_values(self): - a = Analytics(total_documents=42, total_chunks=100, total_topics=5, total_tags=10, database_size_bytes=1024) - assert a.total_documents == 42 - - -class TestAskResult: - def test_basic(self): - r = AskResult(answer="42", sources=[{"id": "1"}]) - assert r.answer == "42" - assert len(r.sources) == 1 - - -class TestGraph: - def test_empty(self): - g = Graph() - assert g.nodes == [] - assert g.edges == [] - - def test_with_data(self): - g = Graph( - nodes=[GraphNode(id="1", label="Doc1")], - edges=[GraphEdge(source="1", target="2", weight=0.8)], - ) - assert len(g.nodes) == 1 - assert g.edges[0].weight == 0.8 - - -class TestSyncResult: - def test_basic(self): - s = SyncResult(connector="obsidian", documents_synced=5) - assert s.connector == "obsidian" - assert s.errors == [] +def test_topic_defaults(): + topic = Topic.model_validate({"id": "a", "name": "A"}) + assert topic.parent_id is None + assert topic.document_count is None diff --git a/src/LibScope.ts b/src/LibScope.ts index 5d88dace..90fc3ca0 100644 --- a/src/LibScope.ts +++ b/src/LibScope.ts @@ -1,146 +1,398 @@ +/** + * The public Node.js API. Every method is a typed call to one operation from + * src/core/operations, so the SDK has the same parameters, defaults and results as the CLI, + * MCP server and REST API. Namespaces are generated from the NAMESPACES table below. + */ import type Database from "better-sqlite3"; -import type { LibScopeConfig } from "./config.js"; -import type { EmbeddingProvider } from "./providers/embedding.js"; -import { loadConfig } from "./config.js"; -import { createDatabase } from "./db/connection.js"; -import { runMigrations, createVectorTable } from "./db/schema.js"; -import { createEmbeddingProvider } from "./providers/index.js"; -import { createLlmProvider } from "./core/rag.js"; -import { getWorkspacePath, DEFAULT_WORKSPACE } from "./core/workspace.js"; -import { - indexDocument, - indexFile, - type IndexDocumentInput, - type IndexFileOptions, - type IndexedDocument, -} from "./core/indexing.js"; -import { searchDocuments, type SearchOptions, type SearchResponse } from "./core/search.js"; +import type { z } from "zod"; +import { ConfigSchema, type LibScopeConfig } from "./config.js"; +import { bootstrap, type Bootstrapped, type ConfigOverrides } from "./core/bootstrap.js"; +import type { Chunker } from "./core/indexing.js"; import { - askQuestion, askQuestionStream, - type RagOptions, - type RagResult, + createLlmProvider, + type LlmProvider, type RagStreamEvent, } from "./core/rag.js"; -import { getStats, type OverviewStats } from "./core/analytics.js"; -import { listDocuments, getDocument, deleteDocument, type Document } from "./core/documents.js"; +import type { EmbeddingProvider } from "./providers/embedding.js"; +import { + backupOperation, + bulkDeleteOperation, + bulkMoveOperation, + bulkRetagOperation, + dedupeOperation, + overviewOperation, + pruneExpiredOperation, + reindexOperation, + restoreOperation, +} from "./core/operations/admin.js"; +import { + popularOperation, + searchAnalyticsOperation, + staleOperation, + topQueriesOperation, +} from "./core/operations/analytics.js"; +import { + addOperation, + deleteDocumentOperation, + documentHistoryOperation, + getDocumentOperation, + listDocumentsOperation, + rateDocumentOperation, + rollbackDocumentOperation, + updateDocumentOperation, +} from "./core/operations/documents.js"; +import { graphOperation } from "./core/operations/graph.js"; +import { + linkDocumentsOperation, + listLinksOperation, + prerequisitesOperation, + unlinkDocumentsOperation, +} from "./core/operations/links.js"; +import { + createPackOperation, + installPackOperation, + listPacksOperation, + removePackOperation, +} from "./core/operations/packs.js"; +import { + addRegistryOperation, + createRegistryOperation, + listRegistriesOperation, + publishPackOperation, + removeRegistryOperation, + searchRegistriesOperation, + syncRegistriesOperation, + unpublishPackOperation, +} from "./core/operations/registries.js"; +import { askOperation, searchOperation, toRagOptions } from "./core/operations/search.js"; +import { + deleteSavedSearchOperation, + listSavedSearchesOperation, + runSavedSearchOperation, + saveSearchOperation, +} from "./core/operations/searches.js"; +import { + addTagsOperation, + listTagsOperation, + removeTagsOperation, + suggestTagsOperation, +} from "./core/operations/tags.js"; +import { + cancelTaskOperation, + getTaskOperation, + listTasksOperation, +} from "./core/operations/tasks.js"; import { - searchBatch, - type BatchSearchRequest, - type BatchSearchResponse, -} from "./core/batch-search.js"; + createTopicOperation, + deleteTopicOperation, + listTopicsOperation, +} from "./core/operations/topics.js"; +import { + createOperationContext, + parseOperationInput, + runOperation, + type Operation, + type OperationContext, + type ProgressEvent, +} from "./core/operations/types.js"; +import { + createWebhookOperation, + deleteWebhookOperation, + listWebhooksOperation, + testWebhookOperation, +} from "./core/operations/webhooks.js"; + +type ConnectorOperations = typeof import("./core/operations/connectors.js"); export interface LibScopeOptions { - /** Workspace name (default: "default"). */ - workspace?: string; - /** Override config values. */ - config?: Partial<LibScopeConfig>; + /** + * Workspace whose database to open. Default: the active workspace (LIBSCOPE_WORKSPACE, + * `.libscope.json` `workspace`, or `libscope workspace use`), the same one the CLI and MCP + * server use. Ignored when `dbPath`, `db` or `database.path` is set. + */ + workspace?: string | undefined; + /** Explicit SQLite database file (":memory:" for an in-memory database). */ + dbPath?: string | undefined; + /** Use an already-open database. It is migrated, and `close()` leaves it open. */ + db?: Database.Database | undefined; + /** Per-section config values merged over the loaded config, e.g. `{ llm: { model: "x" } }`. */ + config?: ConfigOverrides | undefined; + /** + * false: ignore config files, secrets.json and LIBSCOPE_* config variables; start from the + * defaults plus `config`. Default true. + */ + useConfigFile?: boolean | undefined; + /** Embedding provider instance to use instead of the configured one. */ + provider?: EmbeddingProvider | undefined; + /** LLM used by ask/askStream instead of the configured one. */ + llmProvider?: LlmProvider | undefined; + /** Custom chunker used by `add` for inline content and local files. */ + chunker?: Chunker | undefined; +} + +/** Per-call options for long-running methods. */ +export interface RunOptions { + signal?: AbortSignal | undefined; + onProgress?: ((progress: ProgressEvent) => void) | undefined; +} + +/** An operation whose module is loaded on first use (keeps connectors out of `libscope/lite`). */ +interface LazyOperation<Op> { + readonly name: string; + load(): Promise<Op>; +} + +function lazy<Op>(name: string, load: () => Promise<Op>): LazyOperation<Op> { + return { name, load }; +} + +const loadConnectorOperations = (): Promise<ConnectorOperations> => + import("./core/operations/connectors.js"); + +/** Namespace -> method -> operation. Every operation appears here or as a top-level method. */ +const NAMESPACES = { + docs: { + get: getDocumentOperation, + list: listDocumentsOperation, + update: updateDocumentOperation, + delete: deleteDocumentOperation, + rate: rateDocumentOperation, + history: documentHistoryOperation, + rollback: rollbackDocumentOperation, + }, + topics: { + list: listTopicsOperation, + create: createTopicOperation, + delete: deleteTopicOperation, + }, + tags: { + add: addTagsOperation, + remove: removeTagsOperation, + list: listTagsOperation, + suggest: suggestTagsOperation, + }, + links: { + create: linkDocumentsOperation, + delete: unlinkDocumentsOperation, + list: listLinksOperation, + prerequisites: prerequisitesOperation, + graph: graphOperation, + }, + searches: { + save: saveSearchOperation, + list: listSavedSearchesOperation, + run: runSavedSearchOperation, + delete: deleteSavedSearchOperation, + }, + packs: { + install: installPackOperation, + remove: removePackOperation, + list: listPacksOperation, + create: createPackOperation, + }, + registries: { + list: listRegistriesOperation, + add: addRegistryOperation, + remove: removeRegistryOperation, + sync: syncRegistriesOperation, + search: searchRegistriesOperation, + create: createRegistryOperation, + publish: publishPackOperation, + unpublish: unpublishPackOperation, + }, + connectors: { + list: lazy( + "list-connections", + async () => (await loadConnectorOperations()).listConnectionsOperation, + ), + sync: lazy("sync", async () => (await loadConnectorOperations()).syncOperation), + disconnect: lazy( + "disconnect", + async () => (await loadConnectorOperations()).disconnectOperation, + ), + }, + tasks: { + get: getTaskOperation, + cancel: cancelTaskOperation, + list: listTasksOperation, + }, + admin: { + reindex: reindexOperation, + dedupe: dedupeOperation, + backup: backupOperation, + restore: restoreOperation, + pruneExpired: pruneExpiredOperation, + bulkDelete: bulkDeleteOperation, + bulkRetag: bulkRetagOperation, + bulkMove: bulkMoveOperation, + }, + analytics: { + popular: popularOperation, + stale: staleOperation, + topQueries: topQueriesOperation, + searches: searchAnalyticsOperation, + }, + webhooks: { + create: createWebhookOperation, + list: listWebhooksOperation, + delete: deleteWebhookOperation, + test: testWebhookOperation, + }, +} as const; + +/** Operations called by the top-level methods (add, search, ask, askStream, overview). */ +const TOP_LEVEL = [addOperation, searchOperation, askOperation, overviewOperation] as const; + +type AnyOperation = Operation<z.ZodObject, unknown>; +type OperationSource = AnyOperation | LazyOperation<AnyOperation>; +type Resolved<Src> = Src extends LazyOperation<infer Op> ? Op : Src; + +/** Input accepted by an operation (before defaults are applied). */ +export type OperationInput<Op> = Op extends Operation<infer S, unknown> ? z.input<S> : never; +/** Result of an operation. */ +export type OperationOutput<Op> = Op extends Operation<z.ZodObject, infer O> ? Awaited<O> : never; + +// eslint-disable-next-line @typescript-eslint/no-empty-object-type -- "every field optional" test +type AllOptional<T> = {} extends T ? true : false; + +type Method<Op> = + AllOptional<OperationInput<Op>> extends true + ? (input?: OperationInput<Op>, run?: RunOptions) => Promise<OperationOutput<Op>> + : (input: OperationInput<Op>, run?: RunOptions) => Promise<OperationOutput<Op>>; + +type Namespaces = typeof NAMESPACES; +export type NamespaceName = keyof Namespaces; +/** The methods of one namespace, e.g. `LibScopeNamespace<"docs">`. */ +export type LibScopeNamespace<N extends NamespaceName> = NamespaceMethods<Namespaces[N]>; +type NamespaceMethods<T> = { readonly [M in keyof T]: Method<Resolved<T[M]>> }; +/** Input of a namespace method, e.g. `LibScopeInput<"docs", "update">`. */ +export type LibScopeInput<N extends NamespaceName, M extends keyof Namespaces[N]> = OperationInput< + Resolved<Namespaces[N][M]> +>; +/** Result of a namespace method, e.g. `LibScopeOutput<"docs", "get">`. */ +export type LibScopeOutput< + N extends NamespaceName, + M extends keyof Namespaces[N], +> = OperationOutput<Resolved<Namespaces[N][M]>>; + +export type AddInput = OperationInput<typeof addOperation>; +export type AddResult = OperationOutput<typeof addOperation>; +export type SearchInput = OperationInput<typeof searchOperation>; +export type SearchOutput = OperationOutput<typeof searchOperation>; +export type AskInput = OperationInput<typeof askOperation>; +export type AskOutput = OperationOutput<typeof askOperation>; + +/** Operation names reachable from the SDK (used by tests to keep the SDK complete). */ +export function sdkOperationNames(): string[] { + const nested = Object.values(NAMESPACES).flatMap((ns) => + Object.values(ns as Record<string, OperationSource>).map((src) => src.name), + ); + return [...TOP_LEVEL.map((op) => op.name), ...nested]; +} + +/** Config built from the schema defaults only (no files, no environment). */ +function defaultConfig(): LibScopeConfig { + const sections = Object.fromEntries(Object.keys(ConfigSchema.shape).map((key) => [key, {}])); + return ConfigSchema.parse(sections) as unknown as LibScopeConfig; } export class LibScope { - private readonly db: Database.Database; - private readonly embeddingProvider: EmbeddingProvider; - private readonly config: LibScopeConfig; + readonly docs = this.bind(NAMESPACES.docs); + readonly topics = this.bind(NAMESPACES.topics); + readonly tags = this.bind(NAMESPACES.tags); + readonly links = this.bind(NAMESPACES.links); + readonly searches = this.bind(NAMESPACES.searches); + readonly packs = this.bind(NAMESPACES.packs); + readonly registries = this.bind(NAMESPACES.registries); + readonly connectors = this.bind(NAMESPACES.connectors); + readonly tasks = this.bind(NAMESPACES.tasks); + readonly admin = this.bind(NAMESPACES.admin); + readonly analytics = this.bind(NAMESPACES.analytics); + readonly webhooks = this.bind(NAMESPACES.webhooks); + + private readonly ctx: OperationContext; private constructor( - db: Database.Database, - embeddingProvider: EmbeddingProvider, - config: LibScopeConfig, + private readonly boot: Bootstrapped, + private readonly llm: LlmProvider | undefined, + chunker: Chunker | undefined, ) { - this.db = db; - this.embeddingProvider = embeddingProvider; - this.config = config; + this.ctx = createOperationContext({ + db: boot.db, + provider: boot.provider, + config: boot.config, + surface: "sdk", + llm, + chunker, + }); } - /** - * Create a new LibScope instance. Initializes DB, runs migrations, and sets up providers. - */ - static create(options?: LibScopeOptions): LibScope { - const baseConfig = loadConfig(); - const config: LibScopeConfig = { - embedding: { ...baseConfig.embedding, ...options?.config?.embedding }, - llm: { ...baseConfig.llm, ...options?.config?.llm }, - database: { ...baseConfig.database, ...options?.config?.database }, - indexing: { ...baseConfig.indexing, ...options?.config?.indexing }, - logging: { ...baseConfig.logging, ...options?.config?.logging }, - }; - - const workspace = options?.workspace ?? DEFAULT_WORKSPACE; - const dbPath = config.database.path ?? getWorkspacePath(workspace); - - const db = createDatabase(dbPath); - runMigrations(db); - - const embeddingProvider = createEmbeddingProvider(config); - createVectorTable(db, embeddingProvider.dimensions); - - return new LibScope(db, embeddingProvider, config); + /** Open (and migrate) the database and create the providers. Call `close()` when done. */ + static create(options: LibScopeOptions = {}): LibScope { + const boot = bootstrap({ + workspace: options.workspace, + dbPath: options.dbPath, + db: options.db, + config: options.useConfigFile === false ? defaultConfig() : undefined, + configOverrides: options.config, + provider: options.provider, + }); + return new LibScope(boot, options.llmProvider, options.chunker); } - /** Index a document from content. */ - async index(input: IndexDocumentInput): Promise<IndexedDocument> { - return indexDocument(this.db, this.embeddingProvider, input); + /** Add content, a file, a directory, a URL (optionally crawled) or a repository. */ + add(input: string | AddInput, run?: RunOptions): Promise<AddResult> { + const raw = typeof input === "string" ? { source: input } : input; + return runOperation(addOperation, this.context(run), raw); } - /** Index a file from disk. */ - async indexFile(filePath: string, options?: IndexFileOptions): Promise<IndexedDocument> { - return indexFile(this.db, this.embeddingProvider, filePath, options); + /** Search by meaning and keywords, or find content related to a document or chunk. */ + search(input: string | SearchInput, run?: RunOptions): Promise<SearchOutput> { + const raw = typeof input === "string" ? { query: input } : input; + return runOperation(searchOperation, this.context(run), raw); } - /** Search documents. */ - async search(query: string, options?: Omit<SearchOptions, "query">): Promise<SearchResponse> { - return searchDocuments(this.db, this.embeddingProvider, { ...options, query }); + /** Answer a question with the LLM, or (llm.provider "passthrough") return the context. */ + ask(input: string | AskInput, run?: RunOptions): Promise<AskOutput> { + const raw = typeof input === "string" ? { question: input } : input; + return runOperation(askOperation, this.context(run), raw); } - /** Run multiple searches concurrently. */ - async searchBatch(requests: BatchSearchRequest[]): Promise<BatchSearchResponse> { - return searchBatch(this.db, this.embeddingProvider, requests); + /** Like `ask`, but streams the answer tokens. Needs an LLM (no passthrough). */ + async *askStream(input: string | AskInput): AsyncGenerator<RagStreamEvent> { + const raw = typeof input === "string" ? { question: input } : input; + const parsed = parseOperationInput(askOperation, raw); + const llm = this.llm ?? createLlmProvider(this.boot.config, { surface: "sdk" }); + yield* askQuestionStream(this.boot.db, this.boot.provider, llm, toRagOptions(this.ctx, parsed)); } - /** Ask a question using RAG. */ - async ask(question: string, options?: Omit<RagOptions, "question">): Promise<RagResult> { - const llm = createLlmProvider(this.config); - return askQuestion(this.db, this.embeddingProvider, llm, { ...options, question }); + /** Counts, topics, installed packs, index identity and health. */ + overview(): Promise<OperationOutput<typeof overviewOperation>> { + return runOperation(overviewOperation, this.ctx, {}); } - /** Ask a question with streaming response. */ - async *askStream( - question: string, - options?: Omit<RagOptions, "question">, - ): AsyncGenerator<RagStreamEvent> { - const llm = createLlmProvider(this.config); - yield* askQuestionStream(this.db, this.embeddingProvider, llm, { ...options, question }); - } - - /** Get overview stats. */ - stats(): OverviewStats { - return getStats(this.db, this.config.database.path); - } - - /** List documents. */ - list(options?: { - library?: string; - topicId?: string; - sourceType?: string; - dateFrom?: string; - dateTo?: string; - limit?: number; - }): Document[] { - return listDocuments(this.db, options); + /** Close the database (unless it was passed in as `db`). Safe to call twice. */ + close(): void { + this.boot.close(); } - /** Get a document by ID. */ - get(id: string): Document { - return getDocument(this.db, id); + private context(run: RunOptions | undefined): OperationContext { + if (!run) return this.ctx; + return { ...this.ctx, signal: run.signal, onProgress: run.onProgress }; } - /** Delete a document by ID. */ - delete(id: string): void { - deleteDocument(this.db, id); + private async call(src: OperationSource, input: unknown, run?: RunOptions): Promise<unknown> { + const op = "load" in src ? await src.load() : src; + return runOperation(op, this.context(run), input); } - /** Close the database connection. */ - close(): void { - this.db.close(); + private bind<T extends object>(table: T): NamespaceMethods<T> { + const methods: Record<string, (input?: unknown, run?: RunOptions) => Promise<unknown>> = {}; + for (const [key, src] of Object.entries(table)) { + methods[key] = (input, run): Promise<unknown> => + this.call(src as OperationSource, input, run); + } + return Object.freeze(methods) as unknown as NamespaceMethods<T>; } } diff --git a/src/api/adapter.ts b/src/api/adapter.ts new file mode 100644 index 00000000..2e60f151 --- /dev/null +++ b/src/api/adapter.ts @@ -0,0 +1,164 @@ +/** + * HTTP adapter over the operation layer, shared by the REST API and the dashboard server: + * build an operation's input from a request, run it, and map errors to HTTP statuses. + */ +import type { IncomingMessage, ServerResponse } from "node:http"; +import { z } from "zod"; +import { + runOperation, + startOperationTask, + type Operation, + type OperationContext, +} from "../core/operations/index.js"; +import { ConfigError, FetchError, NotFoundError, ValidationError } from "../errors.js"; +import { getLogger } from "../logger.js"; +import { HttpError, parseJsonBody, sendError } from "./middleware.js"; + +/** The subset of JSON Schema the adapter and the OpenAPI generator read. */ +export interface JsonSchema { + type?: string | string[]; + anyOf?: JsonSchema[]; + items?: JsonSchema; + properties?: Record<string, JsonSchema>; + required?: string[]; + [key: string]: unknown; +} + +const schemaCache = new WeakMap<Operation, JsonSchema>(); + +/** JSON Schema of an operation's input (what callers send), without the `$schema` key. */ +export function inputJsonSchema(op: Operation): JsonSchema { + let schema = schemaCache.get(op); + if (!schema) { + schema = { + ...(z.toJSONSchema(op.input, { io: "input", unrepresentable: "any" }) as JsonSchema), + }; + delete schema["$schema"]; + schemaCache.set(op, schema); + } + return schema; +} + +/** The non-null branch of a nullable schema (zod writes `anyOf: [T, {type: "null"}]`). */ +function nonNull(schema: JsonSchema | undefined): JsonSchema | undefined { + if (!schema?.anyOf) return schema; + return schema.anyOf.find((alt) => alt.type !== "null"); +} + +function coerceScalar(raw: string, schema: JsonSchema | undefined): unknown { + const type = nonNull(schema)?.type; + if (type === "integer" || type === "number") { + const n = Number(raw); + return raw.trim() !== "" && Number.isFinite(n) ? n : raw; + } + if (type === "boolean") { + if (raw === "true") return true; + if (raw === "false") return false; + } + return raw; +} + +/** + * Operation input from URL values (query string or path parameters). Each value is converted + * to the type the operation's schema expects: numbers and booleans are parsed, and array + * fields take repeated parameters and/or comma-separated values (`tags=a,b` or `tags=a&tags=b`). + * Values that do not convert are passed through so validation reports them. + */ +export function inputFromParams( + op: Operation, + params: URLSearchParams | Record<string, string>, +): Record<string, unknown> { + const search = params instanceof URLSearchParams ? params : new URLSearchParams(params); + const properties = inputJsonSchema(op).properties ?? {}; + const input: Record<string, unknown> = {}; + for (const name of new Set(search.keys())) { + const schema = nonNull(properties[name]); + if (schema?.type === "array") { + input[name] = search + .getAll(name) + .flatMap((v) => v.split(",")) + .map((v) => v.trim()) + .filter((v) => v !== "") + .map((v) => coerceScalar(v, schema.items)); + } else { + input[name] = coerceScalar(search.get(name) ?? "", schema); + } + } + return input; +} + +/** A decoded URL path segment; a malformed escape is a 400. */ +export function decodeSegment(segment: string): string { + try { + return decodeURIComponent(segment); + } catch { + throw new HttpError(400, "VALIDATION_ERROR", "Malformed URL path"); + } +} + +/** The JSON request body as an object ({} when empty). */ +export async function readJsonObject(req: IncomingMessage): Promise<Record<string, unknown>> { + const body = await parseJsonBody(req); + if (body === null) return {}; + if (typeof body !== "object" || Array.isArray(body)) { + throw new HttpError(400, "VALIDATION_ERROR", "Request body must be a JSON object"); + } + return body as Record<string, unknown>; +} + +export interface OperationResponse { + status: number; + data: unknown; + /** Set for a background task: where to poll it. */ + location?: string | undefined; +} + +/** + * Run an operation for an HTTP request. A long-running operation starts as a background task + * and answers 202 with the task ID; poll `GET {tasksPath}/{taskId}`. + */ +export async function respondWithOperation( + op: Operation, + ctx: OperationContext, + input: Record<string, unknown>, + tasksPath: string, +): Promise<OperationResponse> { + if (op.annotations?.longRunning) { + const task = startOperationTask(op, ctx, input); + return { + status: 202, + data: { taskId: task.id, operation: op.name, status: task.status }, + location: `${tasksPath}/${task.id}`, + }; + } + return { status: 200, data: await runOperation(op, ctx, input) }; +} + +/** HTTP status, code and client-safe message for an error thrown while handling a request. */ +export function describeError(err: unknown): { status: number; code: string; message: string } { + if (err instanceof HttpError) return { status: err.status, code: err.code, message: err.message }; + if (err instanceof ValidationError) return { status: 400, code: err.code, message: err.message }; + if (err instanceof NotFoundError) return { status: 404, code: err.code, message: err.message }; + if (err instanceof FetchError) return { status: 502, code: err.code, message: err.message }; + // Config problems (e.g. no LLM for ask) carry a hint the caller needs; other errors may leak + // internals, so they get a generic message. + if (err instanceof ConfigError) return { status: 500, code: err.code, message: err.message }; + return { status: 500, code: "INTERNAL_ERROR", message: "Internal server error" }; +} + +/** Log an error and send it as `{ error: { code, message } }`. */ +export function sendRequestError( + res: ServerResponse, + err: unknown, + request: { method: string; pathname: string }, +): void { + const { status, code, message } = describeError(err); + const log = getLogger(); + if (status >= 500) log.error({ err, ...request }, "API request failed"); + else log.debug({ err, ...request }, "API request rejected"); + if (res.headersSent) { + res.end(); + return; + } + sendError(res, status, code, message); +} diff --git a/src/api/indexing/repoConfig.ts b/src/api/indexing/repoConfig.ts deleted file mode 100644 index 1d114733..00000000 --- a/src/api/indexing/repoConfig.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { join } from "node:path"; -import { homedir } from "node:os"; -import { readFileSync, mkdirSync } from "node:fs"; -import { LibScopeLite } from "../../lite/core.js"; -import { createDatabase } from "../../db/connection.js"; -import { runMigrations, createVectorTable } from "../../db/schema.js"; -import { LocalEmbeddingProvider } from "../../providers/local.js"; - -export interface RepoEntry { - cloneUrl: string; - branch?: string; - include?: string[]; - exclude?: string[]; -} - -export interface ReposConfig { - repos: Record<string, RepoEntry>; -} - -export function loadReposConfig(): ReposConfig { - const configPath = process.env["LIBSCOPE_REPOS_CONFIG"]; - if (!configPath) { - return { repos: {} }; - } - try { - const raw = readFileSync(configPath, "utf8"); - return JSON.parse(raw) as ReposConfig; - } catch { - return { repos: {} }; - } -} - -export function repoDbPath(repoSlug: string): string { - return join(homedir(), ".libscope", "repos", `${repoSlug}.db`); -} - -export function createRepoLibScope(repoSlug: string): LibScopeLite { - const dbPath = repoDbPath(repoSlug); - mkdirSync(join(homedir(), ".libscope", "repos"), { recursive: true }); - const provider = new LocalEmbeddingProvider(); - const db = createDatabase(dbPath); - runMigrations(db); - try { - createVectorTable(db, provider.dimensions); - } catch { - /* sqlite-vec not loaded */ - } - return new LibScopeLite({ db, provider }); -} diff --git a/src/api/indexing/repoIndexer.ts b/src/api/indexing/repoIndexer.ts deleted file mode 100644 index 86449b32..00000000 --- a/src/api/indexing/repoIndexer.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { execFileSync } from "node:child_process"; -import { readdirSync, readFileSync, rmSync, statSync, existsSync } from "node:fs"; -import { join, relative, extname } from "node:path"; -import { tmpdir } from "node:os"; -import { randomUUID } from "node:crypto"; -import { LibScopeLite } from "../../lite/core.js"; -import { TreeSitterChunker } from "../../lite/chunker-treesitter.js"; -import type { LiteDoc } from "../../lite/types.js"; -import type { RepoEntry } from "./repoConfig.js"; - -const chunker = new TreeSitterChunker(); - -/** - * Absolute path to the git executable, resolved once at module load time. - * Using an absolute path means the OS never consults PATH, eliminating the - * risk of a tampered environment redirecting "git" to a malicious binary. - */ -const GIT_EXECUTABLE = ((): string => { - const candidates = ["/usr/bin/git", "/usr/local/bin/git", "/opt/homebrew/bin/git"]; - for (const candidate of candidates) { - if (existsSync(candidate)) return candidate; - } - throw new Error("git executable not found at any known path"); -})(); - -export interface IndexJobStats { - filesIndexed: number; - chunksCreated: number; - filesSkipped: number; - errors: string[]; - durationMs: number; -} - -/** Map file extension to language string understood by TreeSitterChunker. */ -function detectLanguage(filePath: string): string | undefined { - const ext = extname(filePath).toLowerCase().replace(/^\./, ""); - const EXTENSION_MAP: Record<string, string> = { - ts: "typescript", - tsx: "typescript", - js: "javascript", - jsx: "javascript", - mjs: "javascript", - cjs: "javascript", - py: "python", - cs: "csharp", - cpp: "cpp", - cc: "cpp", - cxx: "cpp", - hpp: "cpp", - h: "cpp", - c: "c", - go: "go", - }; - return EXTENSION_MAP[ext]; -} - -/** Convert a glob-style pattern to a RegExp. Handles ** and * wildcards. */ -function globToRegex(pattern: string): RegExp { - const escaped = pattern - .replaceAll(/[.+^${}()|[\]\\]/g, String.raw`\$&`) - .replaceAll("**", "§DOUBLESTAR§") - .replaceAll("*", "[^/]*") - .replaceAll("§DOUBLESTAR§", ".*"); - return new RegExp(`^${escaped}$`); -} - -/** Recursively list all files under dir, returning paths relative to dir. */ -function walkFiles(dir: string, include?: string[], exclude?: string[]): string[] { - const includeRegexes = include?.map(globToRegex); - const excludeRegexes = exclude?.map(globToRegex); - const results: string[] = []; - - function recurse(current: string): void { - let entries; - try { - entries = readdirSync(current, { withFileTypes: true }); - } catch { - return; - } - for (const entry of entries) { - const full = join(current, entry.name); - const rel = relative(dir, full); - if (entry.isDirectory()) { - // Skip hidden directories - if (entry.name.startsWith(".")) continue; - recurse(full); - } else if (entry.isFile()) { - if (excludeRegexes?.some((rx) => rx.test(rel))) continue; - if (includeRegexes && !includeRegexes.some((rx) => rx.test(rel))) continue; - results.push(rel); - } - } - } - - recurse(dir); - return results; -} - -/** - * Validate a git branch name against the git-check-ref-format allowlist. - * Rejects anything outside [a-zA-Z0-9._\-/] to prevent argument injection. - */ -function validateBranchName(branch: string): void { - if (!/^[a-zA-Z0-9._\-/]+$/.test(branch)) { - throw new Error( - `Invalid branch name "${branch}": only alphanumerics, '.', '_', '-', and '/' are allowed`, - ); - } -} - -/** Clone a repo at branch into a temp directory. Returns the temp dir path. */ -function cloneToTemp(cloneUrl: string, branch?: string): string { - const tempDir = join(tmpdir(), `libscope-repo-${randomUUID()}`); - const args = ["clone", "--depth=1"]; - if (branch) { - validateBranchName(branch); - args.push("--branch", branch); - } - // Use the absolute path resolved at startup — PATH is never consulted. - // execFileSync with an array bypasses the shell entirely. - // The "--" separator prevents git from interpreting the URL as a flag. - args.push("--", cloneUrl, tempDir); - execFileSync(GIT_EXECUTABLE, args, { stdio: "ignore" }); - return tempDir; -} - -/** Chunk a source file with tree-sitter and append resulting LiteDocs. */ -async function chunkAndIndex( - relPath: string, - source: string, - lang: string, - repoSlug: string, - allDocs: LiteDoc[], - stats: IndexJobStats, -): Promise<void> { - try { - const chunks = await chunker.chunk(source, lang); - for (const chunk of chunks) { - allDocs.push({ - title: `${relPath}:L${String(chunk.startLine)}-L${String(chunk.endLine)} [${chunk.nodeType}]`, - content: chunk.content, - library: repoSlug, - url: `repo://${repoSlug}/${relPath}#L${String(chunk.startLine)}`, - }); - stats.chunksCreated++; - } - stats.filesIndexed++; - } catch (err) { - stats.errors.push(`${relPath} (chunking): ${err instanceof Error ? err.message : String(err)}`); - stats.filesSkipped++; - } -} - -/** Read one file from the cloned repo and append its docs to allDocs. */ -async function processFile( - relPath: string, - tempDir: string, - repoSlug: string, - allDocs: LiteDoc[], - stats: IndexJobStats, -): Promise<void> { - const fullPath = join(tempDir, relPath); - let source: string; - try { - const stat = statSync(fullPath); - if (!stat.isFile()) { - stats.filesSkipped++; - return; - } - source = readFileSync(fullPath, "utf8"); - } catch (err) { - stats.filesSkipped++; - stats.errors.push(`${relPath}: ${err instanceof Error ? err.message : String(err)}`); - return; - } - - const lang = detectLanguage(relPath); - if (lang && chunker.supports(lang)) { - await chunkAndIndex(relPath, source, lang, repoSlug, allDocs, stats); - } else { - // Non-code file or unsupported language — index as raw text - allDocs.push({ - title: relPath, - content: source, - library: repoSlug, - url: `repo://${repoSlug}/${relPath}`, - }); - stats.chunksCreated++; - stats.filesIndexed++; - } -} - -export async function indexRepo( - libscope: LibScopeLite, - repoSlug: string, - entry: RepoEntry, - opts: { branch?: string; files?: string[] }, -): Promise<IndexJobStats> { - const start = Date.now(); - const stats: IndexJobStats = { - filesIndexed: 0, - chunksCreated: 0, - filesSkipped: 0, - errors: [], - durationMs: 0, - }; - - const branch = opts.branch ?? entry.branch; - const tempDir = cloneToTemp(entry.cloneUrl, branch); - - try { - const isIncremental = opts.files !== undefined && opts.files.length > 0; - if (!isIncremental) { - libscope.deleteByLibrary(repoSlug); - } - - const filesToIndex = isIncremental - ? opts.files! - : walkFiles(tempDir, entry.include, entry.exclude); - - const allDocs: LiteDoc[] = []; - for (const relPath of filesToIndex) { - await processFile(relPath, tempDir, repoSlug, allDocs, stats); - } - - await libscope.indexBatch(allDocs, { concurrency: 4 }); - } finally { - rmSync(tempDir, { recursive: true, force: true }); - stats.durationMs = Date.now() - start; - } - - return stats; -} diff --git a/src/api/middleware.ts b/src/api/middleware.ts index 7776fed2..d3e44b6e 100644 --- a/src/api/middleware.ts +++ b/src/api/middleware.ts @@ -1,24 +1,52 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import { timingSafeEqual } from "node:crypto"; +import { LibScopeError } from "../errors.js"; + +/** An error with its own HTTP status (bad body, payload too large, forbidden origin). */ +export class HttpError extends LibScopeError { + constructor( + public readonly status: number, + code: string, + message: string, + ) { + super(message, code); + this.name = "HttpError"; + } +} + +const READ_METHODS = new Set(["GET", "HEAD", "OPTIONS"]); -/** Set CORS headers and handle OPTIONS preflight. Returns true if request was handled (preflight). */ +/** True when `origin` is the origin the request was sent to (same scheme-less host and port). */ +function isSameOrigin(req: IncomingMessage, origin: string): boolean { + try { + return new URL(origin).host === req.headers.host; + } catch { + return false; + } +} + +/** + * Set CORS headers and answer OPTIONS preflight. Returns true when the request was handled. + * `origins` lists the allowed browser origins. "*" allows any origin to read (GET) only: + * write methods are offered only to origins listed by name. + */ export function corsMiddleware( req: IncomingMessage, res: ServerResponse, origins: string[], ): boolean { - const origin = req.headers["origin"] ?? "*"; - const allowedOrigin = origins.includes("*") ? "*" : origins.includes(origin) ? origin : ""; - - if (allowedOrigin) { - res.setHeader("Access-Control-Allow-Origin", allowedOrigin); + const origin = req.headers.origin; + if (origin !== undefined && origins.includes(origin)) { + res.setHeader("Access-Control-Allow-Origin", origin); + res.setHeader("Vary", "Origin"); + res.setHeader("Access-Control-Allow-Methods", "GET, POST, PATCH, PUT, DELETE, OPTIONS"); + } else if (origins.includes("*")) { + res.setHeader("Access-Control-Allow-Origin", "*"); + res.setHeader("Access-Control-Allow-Methods", "GET, OPTIONS"); } - res.setHeader("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS"); res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization"); res.setHeader("Access-Control-Max-Age", "86400"); - setSecurityHeaders(res); - if (req.method === "OPTIONS") { res.writeHead(204); res.end(); @@ -27,16 +55,39 @@ export function corsMiddleware( return false; } -/** Set standard security response headers. */ -export function setSecurityHeaders(res: ServerResponse): void { +/** + * Reject a write request (POST, PATCH, PUT, DELETE) sent by a browser page from another origin + * that is not listed by name in `origins`. Browsers send such requests without a preflight + * when they have no body or a "simple" content type, so CORS headers alone do not stop them. + * Returns true when the request was rejected (403 sent). + */ +export function rejectForeignWrite( + req: IncomingMessage, + res: ServerResponse, + origins: string[], +): boolean { + const origin = req.headers.origin; + if (READ_METHODS.has(req.method ?? "GET") || origin === undefined) return false; + if (origins.includes(origin) || isSameOrigin(req, origin)) return false; + sendError(res, 403, "FORBIDDEN_ORIGIN", "Write requests from this origin are not allowed"); + return true; +} + +/** + * Set standard security response headers. `contentSecurityPolicy` false leaves CSP unset + * (the dashboard loads d3 from a CDN). + */ +export function setSecurityHeaders(res: ServerResponse, contentSecurityPolicy = true): void { res.setHeader("X-Content-Type-Options", "nosniff"); res.setHeader("X-Frame-Options", "DENY"); res.setHeader("X-XSS-Protection", "1; mode=block"); res.setHeader("Referrer-Policy", "strict-origin-when-cross-origin"); - res.setHeader( - "Content-Security-Policy", - "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:", - ); + if (contentSecurityPolicy) { + res.setHeader( + "Content-Security-Policy", + "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:", + ); + } } /** Maximum request body size in bytes (default 1 MB). */ @@ -138,7 +189,7 @@ export function checkApiKey(req: IncomingMessage, res: ServerResponse): boolean return true; } -/** Parse the request body as JSON. Returns the parsed object or null on failure. */ +/** Parse the request body as JSON. Resolves null for an empty body; rejects with HttpError. */ export async function parseJsonBody( req: IncomingMessage, maxBytes: number = MAX_BODY_SIZE, @@ -150,7 +201,9 @@ export async function parseJsonBody( received += chunk.length; if (received > maxBytes) { req.destroy(); - reject(new Error(`Request body too large (max ${maxBytes} bytes)`)); + reject( + new HttpError(413, "PAYLOAD_TOO_LARGE", `Request body too large (max ${maxBytes} bytes)`), + ); return; } chunks.push(chunk); @@ -164,7 +217,7 @@ export async function parseJsonBody( try { resolve(JSON.parse(raw)); } catch { - reject(new Error("Invalid JSON body")); + reject(new HttpError(400, "INVALID_JSON", "Request body contains invalid JSON")); } }); req.on("error", reject); diff --git a/src/api/openapi.ts b/src/api/openapi.ts index e61b4f22..19d90409 100644 --- a/src/api/openapi.ts +++ b/src/api/openapi.ts @@ -1,497 +1,211 @@ -export const OPENAPI_SPEC = { - openapi: "3.0.3", - info: { - title: "LibScope REST API", - version: "1.0.0", - description: "AI-powered knowledge base with semantic search, document indexing, and RAG Q&A.", - }, - servers: [{ url: "http://localhost:3378", description: "Local development" }], - paths: { - "/api/v1/search": { - get: { - summary: "Semantic search", - operationId: "searchDocuments", - parameters: [ - { - name: "q", - in: "query", - required: true, - schema: { type: "string" }, - description: "Search query", - }, - { - name: "topic", - in: "query", - schema: { type: "string" }, - description: "Filter by topic", - }, - { name: "tag", in: "query", schema: { type: "string" }, description: "Filter by tag" }, - { - name: "limit", - in: "query", - schema: { type: "integer", default: 10 }, - description: "Max results", - }, - ], - responses: { - "200": { - description: "Search results", - content: { - "application/json": { - schema: { $ref: "#/components/schemas/SearchResponse" }, - example: { - data: { - results: [ - { - documentId: "abc123", - title: "Getting Started", - content: "...", - score: 0.95, - }, - ], - totalCount: 1, - }, - meta: { took: 42 }, - }, - }, - }, - }, - }, - }, - }, - "/api/v1/documents": { - get: { - summary: "List documents with filters", - operationId: "listDocuments", - parameters: [ - { name: "topic", in: "query", schema: { type: "string" } }, - { name: "tag", in: "query", schema: { type: "string" } }, - { name: "limit", in: "query", schema: { type: "integer", default: 50 } }, - { name: "offset", in: "query", schema: { type: "integer", default: 0 } }, - ], - responses: { - "200": { - description: "List of documents", - content: { - "application/json": { schema: { $ref: "#/components/schemas/DocumentListResponse" } }, - }, - }, - }, - }, - post: { - summary: "Index a new document", - operationId: "indexDocument", - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/IndexDocumentRequest" }, - example: { content: "# Guide\nHow to...", title: "My Guide", topic: "tutorials" }, - }, - }, - }, - responses: { - "201": { - description: "Document indexed", - content: { - "application/json": { schema: { $ref: "#/components/schemas/DocumentResponse" } }, - }, - }, +/** + * OpenAPI 3.1 document generated from the REST routes. Operation routes take their summary, + * description and parameter schemas from the operation (zod -> JSON Schema), so the spec + * cannot drift from what the server accepts. + */ +import { createRequire } from "node:module"; +import { inputJsonSchema, type JsonSchema } from "./adapter.js"; +import type { ApiRoute } from "./routes.js"; + +const pkg = createRequire(import.meta.url)("../../package.json") as { version: string }; + +const ref = (name: string): { $ref: string } => ({ $ref: `#/components/responses/${name}` }); + +const ERROR_RESPONSES = { + "400": ref("BadRequest"), + "401": ref("Unauthorized"), + "404": ref("NotFound"), + "429": ref("RateLimited"), + "500": ref("InternalError"), +}; + +const json = (schema: unknown): { "application/json": { schema: unknown } } => ({ + "application/json": { schema }, +}); + +const SUCCESS = { + description: "Success. `data` holds the operation result.", + content: json({ $ref: "#/components/schemas/Success" }), +}; + +const TASK_ACCEPTED = { + description: + "Started as a background task. Poll GET /api/v1/tasks/{taskId}; when it completes, `result` holds the JSON-encoded operation result.", + headers: { Location: { description: "URL of the task", schema: { type: "string" } } }, + content: json({ $ref: "#/components/schemas/TaskAccepted" }), +}; + +function errorResponse(description: string): Record<string, unknown> { + return { description, content: json({ $ref: "#/components/schemas/Error" }) }; +} + +const COMPONENTS = { + schemas: { + Success: { + type: "object", + required: ["data"], + properties: { + data: { description: "Operation result" }, + meta: { + type: "object", + properties: { took: { type: "integer", description: "Handling time in milliseconds" } }, }, }, }, - "/api/v1/documents/{id}": { - get: { - summary: "Get a single document", - operationId: "getDocument", - parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], - responses: { - "200": { - description: "Document found", - content: { - "application/json": { schema: { $ref: "#/components/schemas/DocumentResponse" } }, - }, - }, - "404": { description: "Document not found" }, - }, - }, - delete: { - summary: "Delete a document", - operationId: "deleteDocument", - parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], - responses: { - "200": { description: "Document deleted" }, - "404": { description: "Document not found" }, - }, - }, - }, - "/api/v1/documents/url": { - post: { - summary: "Index document from URL (with optional spidering)", - operationId: "indexFromUrl", - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/IndexFromUrlRequest" }, - examples: { - single: { - summary: "Single URL", - value: { url: "https://example.com/page", topic: "guides" }, - }, - spider: { - summary: "Spider mode", - value: { - url: "https://docs.example.com", - spider: true, - maxPages: 50, - maxDepth: 2, - }, - }, - }, - }, - }, - }, - responses: { - "201": { - description: - "Document(s) indexed. Returns DocumentResponse for single-URL mode, SpiderResponse for spider mode.", - content: { - "application/json": { - schema: { - oneOf: [ - { $ref: "#/components/schemas/DocumentResponse" }, - { $ref: "#/components/schemas/SpiderResponse" }, - ], - }, - }, - }, + TaskAccepted: { + type: "object", + required: ["data"], + properties: { + data: { + type: "object", + required: ["taskId", "operation", "status"], + properties: { + taskId: { type: "string" }, + operation: { type: "string" }, + status: { type: "string", enum: ["pending", "running", "completed", "failed"] }, }, }, + meta: { $ref: "#/components/schemas/Success/properties/meta" }, }, }, - "/api/v1/ask": { - post: { - summary: "RAG Q&A — ask a question", - operationId: "askQuestion", - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/AskRequest" }, - example: { question: "How do I configure authentication?", topic: "security" }, - }, - }, - }, - responses: { - "200": { - description: "Answer with sources", - content: { - "application/json": { schema: { $ref: "#/components/schemas/AskResponse" } }, - }, - }, - }, - }, - }, - "/api/v1/topics": { - get: { - summary: "List all topics", - operationId: "listTopics", - responses: { - "200": { - description: "List of topics", - content: { - "application/json": { schema: { $ref: "#/components/schemas/TopicListResponse" } }, - }, - }, - }, - }, - post: { - summary: "Create a topic", - operationId: "createTopic", - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/CreateTopicRequest" }, - example: { name: "tutorials", parentId: null }, - }, - }, - }, - responses: { - "201": { - description: "Topic created", - content: { - "application/json": { schema: { $ref: "#/components/schemas/TopicResponse" } }, - }, - }, - }, - }, - }, - "/api/v1/tags": { - get: { - summary: "List all tags", - operationId: "listTags", - responses: { - "200": { - description: "List of tags", - content: { - "application/json": { schema: { $ref: "#/components/schemas/TagListResponse" } }, - }, - }, - }, - }, - }, - "/api/v1/documents/{id}/tags": { - post: { - summary: "Add tags to a document", - operationId: "addTagsToDocument", - parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], - requestBody: { - required: true, - content: { - "application/json": { - schema: { $ref: "#/components/schemas/AddTagsRequest" }, - example: { tags: ["typescript", "api"] }, - }, - }, - }, - responses: { - "200": { description: "Tags added" }, - }, - }, - }, - "/api/v1/stats": { - get: { - summary: "Usage statistics", - operationId: "getStats", - responses: { - "200": { - description: "Statistics", - content: { - "application/json": { schema: { $ref: "#/components/schemas/StatsResponse" } }, - }, - }, - }, - }, - }, - "/api/v1/health": { - get: { - summary: "Health check", - operationId: "healthCheck", - responses: { - "200": { - description: "Service health", - content: { - "application/json": { - example: { data: { status: "ok", docCount: 42, dbSize: 1048576 } }, - }, - }, - }, - }, - }, - }, - "/openapi.json": { - get: { - summary: "OpenAPI 3.0 specification", - operationId: "getOpenApiSpec", - responses: { - "200": { description: "OpenAPI spec JSON" }, + Error: { + type: "object", + required: ["error"], + properties: { + error: { + type: "object", + required: ["code", "message"], + properties: { code: { type: "string" }, message: { type: "string" } }, }, }, }, }, - components: { - schemas: { - SearchResponse: { - type: "object", - properties: { - data: { - type: "object", - properties: { - results: { - type: "array", - items: { - type: "object", - properties: { - documentId: { type: "string" }, - title: { type: "string" }, - content: { type: "string" }, - score: { type: "number" }, - }, - }, - }, - totalCount: { type: "integer" }, - }, - }, - meta: { type: "object", properties: { took: { type: "number" } } }, - }, - }, - DocumentListResponse: { - type: "object", - properties: { data: { type: "array", items: { $ref: "#/components/schemas/Document" } } }, - }, - DocumentResponse: { - type: "object", - properties: { data: { $ref: "#/components/schemas/Document" } }, - }, - Document: { - type: "object", - properties: { - id: { type: "string" }, - title: { type: "string" }, - content: { type: "string" }, - sourceType: { type: "string" }, - library: { type: "string", nullable: true }, - topicId: { type: "string", nullable: true }, - createdAt: { type: "string", format: "date-time" }, - updatedAt: { type: "string", format: "date-time" }, - }, - }, - IndexDocumentRequest: { - type: "object", - required: ["content", "title"], - properties: { - content: { type: "string" }, - title: { type: "string" }, - topic: { type: "string" }, - tags: { type: "array", items: { type: "string" } }, - source: { type: "string" }, - }, - }, - IndexFromUrlRequest: { - type: "object", - required: ["url"], - properties: { - url: { type: "string", format: "uri" }, - topic: { type: "string" }, - spider: { - type: "boolean", - description: "When true, crawl linked pages starting from the URL (BFS spider mode).", - }, - maxPages: { - type: "integer", - minimum: 1, - description: - "Maximum total pages to fetch in spider mode (default: 25, hard cap: 200).", - }, - maxDepth: { - type: "integer", - minimum: 0, - description: - "Maximum hop depth from the seed URL in spider mode (default: 2, hard cap: 5).", - }, - sameDomain: { - type: "boolean", - description: "Only follow links sharing the seed hostname (default: true).", - }, - pathPrefix: { - type: "string", - description: "Only follow links whose path starts with this prefix (e.g. '/docs/').", - }, - excludePatterns: { - type: "array", - items: { type: "string" }, - description: "Glob patterns for URLs to skip during spidering (e.g. ['*/changelog*']).", - }, - }, - }, - SpiderResponse: { - type: "object", - properties: { - documents: { - type: "array", - items: { - type: "object", - properties: { - id: { type: "string" }, - title: { type: "string" }, - url: { type: "string" }, - }, - }, - }, - pagesFetched: { type: "integer", description: "Pages successfully fetched and indexed." }, - pagesCrawled: { type: "integer", description: "Total pages attempted." }, - pagesSkipped: { type: "integer", description: "Pages skipped by filters or robots.txt." }, - errors: { - type: "array", - items: { - type: "object", - properties: { - url: { type: "string" }, - error: { type: "string" }, - }, - }, - }, - abortReason: { - type: "string", - nullable: true, - enum: ["maxPages", "timeout", null], - description: "Set if the crawl was aborted early.", - }, - }, - }, - AskRequest: { - type: "object", - required: ["question"], - properties: { - question: { type: "string" }, - topic: { type: "string" }, - }, - }, - AskResponse: { - type: "object", - properties: { - data: { - type: "object", - properties: { - answer: { type: "string" }, - sources: { type: "array", items: { type: "object" } }, - model: { type: "string" }, - }, - }, - }, - }, - CreateTopicRequest: { - type: "object", - required: ["name"], - properties: { - name: { type: "string" }, - parentId: { type: "string", nullable: true }, - }, - }, - TopicResponse: { - type: "object", - properties: { data: { type: "object" } }, - }, - TopicListResponse: { - type: "object", - properties: { data: { type: "array", items: { type: "object" } } }, - }, - TagListResponse: { - type: "object", - properties: { data: { type: "array", items: { type: "object" } } }, - }, - AddTagsRequest: { - type: "object", - required: ["tags"], - properties: { tags: { type: "array", items: { type: "string" } } }, - }, - StatsResponse: { - type: "object", - properties: { data: { $ref: "#/components/schemas/Stats" } }, - }, - Stats: { - type: "object", - properties: { - totalDocuments: { type: "integer" }, - totalChunks: { type: "integer" }, - totalTopics: { type: "integer" }, - databaseSizeBytes: { type: "integer" }, - totalSearches: { type: "integer" }, - avgLatencyMs: { type: "number" }, - }, - }, + responses: { + BadRequest: errorResponse("Invalid input (code VALIDATION_ERROR or INVALID_JSON)"), + Unauthorized: errorResponse("Missing or wrong API key (only when LIBSCOPE_API_KEY is set)"), + NotFound: errorResponse("Route or resource not found"), + RateLimited: errorResponse("More than 120 requests per minute from this address"), + InternalError: errorResponse("Server error"), + }, + securitySchemes: { + bearerAuth: { + type: "http", + scheme: "bearer", + description: "Required only when the server has LIBSCOPE_API_KEY set", }, }, -} as const; +}; + +/** "/api/v1/documents/:documentId" -> "/api/v1/documents/{documentId}" */ +export function toOpenApiPath(path: string): string { + return path + .split("/") + .map((segment) => (segment.startsWith(":") ? `{${segment.slice(1)}}` : segment)) + .join("/"); +} + +/** "get-document" -> "getDocument" */ +function camelCase(name: string): string { + return name + .split("-") + .map((part, i) => (i === 0 ? part : part.charAt(0).toUpperCase() + part.slice(1))) + .join(""); +} + +function pathParamNames(path: string): string[] { + return path + .split("/") + .filter((s) => s.startsWith(":")) + .map((s) => s.slice(1)); +} + +function parameter( + name: string, + location: "path" | "query", + schema: JsonSchema | undefined, + required: boolean, +): Record<string, unknown> { + const isArray = schema?.type === "array"; + return { + name, + in: location, + required, + ...(typeof schema?.description === "string" ? { description: schema.description } : {}), + schema: schema ?? { type: "string" }, + ...(isArray ? { style: "form", explode: true } : {}), + }; +} + +/** The OpenAPI operation object for a route generated from an operation. */ +function operationObject(route: ApiRoute, op: NonNullable<ApiRoute["operation"]>): unknown { + const schema = inputJsonSchema(op); + const properties = schema.properties ?? {}; + const required = new Set(schema.required ?? []); + const inPath = pathParamNames(route.path); + const parameters = inPath.map((name) => parameter(name, "path", properties[name], true)); + const rest = Object.keys(properties).filter((name) => !inPath.includes(name)); + + let requestBody: unknown; + if (route.method === "GET" || route.method === "DELETE") { + for (const name of rest) { + parameters.push(parameter(name, "query", properties[name], required.has(name))); + } + } else if (rest.length > 0) { + const bodyRequired = rest.filter((name) => required.has(name)); + requestBody = { + required: bodyRequired.length > 0, + content: json({ + type: "object", + properties: Object.fromEntries(rest.map((name) => [name, properties[name]])), + ...(bodyRequired.length > 0 ? { required: bodyRequired } : {}), + }), + }; + } + + return { + operationId: camelCase(op.name), + tags: [op.group], + summary: op.summary, + ...(op.description ? { description: op.description } : {}), + ...(parameters.length > 0 ? { parameters } : {}), + ...(requestBody ? { requestBody } : {}), + responses: { + ...(op.annotations?.longRunning ? { "202": TASK_ACCEPTED } : { "200": SUCCESS }), + ...ERROR_RESPONSES, + }, + }; +} + +function fixedRouteObject(route: ApiRoute): unknown { + return { + operationId: route.path.endsWith(".json") ? "getOpenApiSpec" : "health", + tags: ["meta"], + summary: route.summary, + responses: { "200": SUCCESS, "429": ERROR_RESPONSES["429"] }, + }; +} + +/** Build the OpenAPI document for `routes`. */ +export function buildOpenApiSpec(routes: readonly ApiRoute[]): Record<string, unknown> { + const paths: Record<string, Record<string, unknown>> = {}; + for (const route of routes) { + const path = toOpenApiPath(route.path); + paths[path] ??= {}; + paths[path][route.method.toLowerCase()] = route.operation + ? operationObject(route, route.operation) + : fixedRouteObject(route); + } + return { + openapi: "3.1.0", + info: { + title: "LibScope REST API", + version: pkg.version, + description: + "Every /api/v1 route runs one LibScope operation (the same operations as the CLI, MCP server and SDK). Responses are `{ data, meta }`; errors are `{ error: { code, message } }`. Long-running operations answer 202 with a task ID.", + }, + servers: [{ url: "http://localhost:3378", description: "Default `libscope serve api`" }], + security: [{}, { bearerAuth: [] }], + paths, + components: COMPONENTS, + }; +} diff --git a/src/api/routes.ts b/src/api/routes.ts index b43d6d82..b1c76bcc 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -1,1284 +1,159 @@ +/** + * REST routes. Every route under /api/v1 is generated from an operation that declares `http`; + * the only hand-written routes are the OpenAPI document and the health check. + */ import type { IncomingMessage, ServerResponse } from "node:http"; -import { URL } from "node:url"; -import { randomUUID } from "node:crypto"; -import type Database from "better-sqlite3"; import { performance } from "node:perf_hooks"; -import type { EmbeddingProvider } from "../providers/embedding.js"; import { - searchDocuments, - listDocuments, - getDocument, - indexDocument, - deleteDocument, - updateDocument, - listTopics, - createTopic, - listTags, - addTagsToDocument, - suggestTags, - getStats, - getSearchAnalytics, - getKnowledgeGaps, - fetchAndConvert, - askQuestion, - askQuestionStream, - createLlmProvider, - createLink, - getDocumentLinks, - deleteLink, - createSavedSearch, - listSavedSearches, - runSavedSearch, - deleteSavedSearch, - bulkDelete, - bulkRetag, - bulkMove, - searchBatch, -} from "../core/index.js"; -import type { LinkType, BulkSelector, BatchSearchRequest } from "../core/index.js"; -import { loadConfig } from "../config.js"; -import { DocumentNotFoundError, FetchError, LibScopeError } from "../errors.js"; -import { getLogger } from "../logger.js"; -import { parseJsonBody, sendJson, sendError } from "./middleware.js"; -import { OPENAPI_SPEC } from "./openapi.js"; -import { getConnectorStatus, getSyncHistory } from "../connectors/sync-tracker.js"; + OPERATIONS, + type HttpMethod, + type Operation, + type OperationContext, +} from "../core/operations/index.js"; +import { HttpError, sendJson } from "./middleware.js"; import { - createWebhook, - listWebhooks, - deleteWebhook, - getWebhook, - buildPayload, - signPayload, - redactWebhook, - validateWebhookUrlSsrf, -} from "../core/webhooks.js"; -import type { WebhookEvent } from "../core/webhooks.js"; -import { loadScheduleEntries } from "../core/scheduler.js"; -import { spiderUrl } from "../core/spider.js"; -import type { SpiderOptions, SpiderStats } from "../core/spider.js"; -import { loadReposConfig, createRepoLibScope } from "./indexing/repoConfig.js"; -import { indexRepo, type IndexJobStats } from "./indexing/repoIndexer.js"; - -// --------------------------------------------------------------------------- -// Shared types -// --------------------------------------------------------------------------- - -/** Context passed to every route handler. */ -interface RouteContext { - req: IncomingMessage; - res: ServerResponse; - db: Database.Database; - provider: EmbeddingProvider; - url: URL; - start: number; -} - -// --------------------------------------------------------------------------- -// Repo index job registry -// --------------------------------------------------------------------------- - -interface IndexJob { - jobId: string; - repoSlug: string; - status: "queued" | "running" | "completed" | "failed"; - startedAt: string; - completedAt?: string; - stats?: IndexJobStats; - error?: string; -} - -const repoIndexJobs = new Map<string, IndexJob>(); - -// --------------------------------------------------------------------------- -// URL / path helpers -// --------------------------------------------------------------------------- - -function parseUrl(req: IncomingMessage): URL { - return new URL(req.url ?? "/", `http://${req.headers["host"] ?? "localhost"}`); -} - -function extractPathSegments(pathname: string): string[] { - return pathname.split("/").filter(Boolean); -} - -/** Match a path like /api/v1/documents/:id and return the id, or null. */ -function matchDocumentId(segments: string[]): string | null { - // ["api", "v1", "documents", "<id>"] - if ( - segments.length === 4 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "documents" - ) { - const id = segments[3]; - // Exclude sub-paths that are named routes - if (id === "url") return null; - return id ?? null; - } - return null; -} - -/** Match /api/v1/documents/:id/tags */ -function matchDocumentTags(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "documents" && - segments[4] === "tags" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/documents/:id/suggest-tags */ -function matchDocumentSuggestTags(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "documents" && - segments[4] === "suggest-tags" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/documents/:id/links */ -function matchDocumentLinks(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "documents" && - segments[4] === "links" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/links/:id */ -function matchLinkId(segments: string[]): string | null { - if ( - segments.length === 4 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "links" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/searches/:id */ -function matchSearchId(segments: string[]): string | null { - if ( - segments.length === 4 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "searches" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/searches/:id/run */ -function matchSearchRun(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "searches" && - segments[4] === "run" - ) { - return segments[3] ?? null; - } - return null; -} - -/** Match /api/v1/webhooks/:id */ -function matchWebhookId(segments: string[]): string | null { - if ( - segments.length === 4 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "webhooks" - ) { - const id = segments[3]; - return id ?? null; - } - return null; + decodeSegment, + inputFromParams, + readJsonObject, + respondWithOperation, + sendRequestError, +} from "./adapter.js"; +import { buildOpenApiSpec } from "./openapi.js"; + +export const API_PREFIX = "/api/v1"; +export const OPENAPI_PATH = "/openapi.json"; +const TASKS_PATH = `${API_PREFIX}/tasks`; + +export interface ApiRoute { + method: HttpMethod; + /** Full path with `:name` parameters, e.g. "/api/v1/documents/:documentId". */ + path: string; + summary: string; + /** The operation this route runs; absent for the fixed routes (spec, health). */ + operation?: Operation | undefined; + /** Response for a fixed route (no envelope for the OpenAPI document). */ + respond?: ((res: ServerResponse, start: number) => void) | undefined; } -/** Match /api/v1/webhooks/:id/test */ -function matchWebhookTest(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "webhooks" && - segments[4] === "test" - ) { - return segments[3] ?? null; - } - return null; -} - -// --------------------------------------------------------------------------- -// Parameter parsing helpers -// --------------------------------------------------------------------------- - -/** Parse an optional integer search param, returning NaN if absent/invalid. */ -function parseOptionalInt(url: URL, name: string): number { - const raw = url.searchParams.get(name); - return raw ? Number.parseInt(raw, 10) : Number.NaN; -} - -/** Parse an optional integer param clamped to [min, max], with a fallback default. */ -function parseClampedInt( - url: URL, - name: string, - min: number, - max: number, - fallback: number, -): number { - const parsed = parseOptionalInt(url, name); - return Number.isNaN(parsed) ? fallback : Math.max(min, Math.min(parsed, max)); -} - -/** Parse an optional integer param clamped to [min, max], returning undefined if absent. */ -function parseClampedIntOrUndefined( - url: URL, - name: string, - min: number, - max: number, -): number | undefined { - const parsed = parseOptionalInt(url, name); - return Number.isNaN(parsed) ? undefined : Math.max(min, Math.min(parsed, max)); -} - -/** Compute elapsed milliseconds since start. */ function elapsed(start: number): number { return Math.round(performance.now() - start); } -/** Parse a JSON request body, returning null-guarded record or sending a 400 error. */ -async function requireJsonBody( - req: IncomingMessage, - res: ServerResponse, -): Promise<Record<string, unknown> | null> { - const body = await parseJsonBody(req); - if (!body || typeof body !== "object") { - sendError(res, 400, "VALIDATION_ERROR", "Request body must be a JSON object"); - return null; - } - return body as Record<string, unknown>; -} - -// --------------------------------------------------------------------------- -// Route handlers — each has low cognitive complexity -// --------------------------------------------------------------------------- - -function handleOpenApiSpec(ctx: RouteContext): void { - ctx.res.writeHead(200, { "Content-Type": "application/json" }); - ctx.res.end(JSON.stringify(OPENAPI_SPEC)); -} - -function handleHealthCheck(ctx: RouteContext): void { - const config = loadConfig(); - const dbPath = config.database?.path; - const stats = getStats(ctx.db, dbPath); - sendJson( - ctx.res, - 200, - { status: "ok", docCount: stats.totalDocuments, dbSize: stats.databaseSizeBytes }, - elapsed(ctx.start), - ); -} - -async function handleSearch(ctx: RouteContext): Promise<void> { - const q = ctx.url.searchParams.get("q") ?? ""; - if (!q) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Query parameter 'q' is required"); - return; - } - if (q.length > 10_000) { - sendError( - ctx.res, - 400, - "VALIDATION_ERROR", - "Query parameter 'q' exceeds maximum length (10000)", - ); - return; - } - const topic = ctx.url.searchParams.get("topic") ?? undefined; - const source = ctx.url.searchParams.get("source") ?? undefined; - const tag = ctx.url.searchParams.get("tag") ?? undefined; - const limit = parseClampedIntOrUndefined(ctx.url, "limit", 1, 1000); - const offsetParsed = parseOptionalInt(ctx.url, "offset"); - const offset = Number.isNaN(offsetParsed) ? undefined : Math.max(0, offsetParsed); - const tags = tag ? [tag] : undefined; - const maxChunksPerDocument = parseClampedIntOrUndefined(ctx.url, "maxChunksPerDocument", 1, 100); - - const result = await searchDocuments(ctx.db, ctx.provider, { - query: q, - topic, - tags, - source, - limit, - offset, - maxChunksPerDocument, - }); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); -} - -async function handleBatchSearch(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (!Array.isArray(b["requests"])) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'requests' must be an array"); - return; - } - const result = await searchBatch(ctx.db, ctx.provider, b["requests"] as BatchSearchRequest[]); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); -} - -function handleListDocuments(ctx: RouteContext): void { - const topicId = ctx.url.searchParams.get("topic") ?? undefined; - const limit = parseClampedInt(ctx.url, "limit", 1, 1000, 100); - const docs = listDocuments(ctx.db, { topicId, limit }); - sendJson(ctx.res, 200, docs, elapsed(ctx.start)); -} - -async function handleCreateDocument(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (typeof b["content"] !== "string" || typeof b["title"] !== "string") { - sendError( - ctx.res, - 400, - "VALIDATION_ERROR", - "Fields 'content' and 'title' are required strings", - ); - return; - } - - const topicId = typeof b["topic"] === "string" ? b["topic"] : undefined; - const source = typeof b["source"] === "string" ? b["source"] : undefined; - const sourceType = - source === "library" || - source === "topic" || - source === "manual" || - source === "model-generated" - ? source - : "manual"; - - const doc = await indexDocument(ctx.db, ctx.provider, { - content: b["content"], - title: b["title"], - sourceType, - topicId, - }); - - // Add tags if provided - if (Array.isArray(b["tags"])) { - const tagNames = (b["tags"] as unknown[]).filter((t): t is string => typeof t === "string"); - if (tagNames.length > 0) { - addTagsToDocument(ctx.db, doc.id, tagNames); - } - } - - sendJson(ctx.res, 201, doc, elapsed(ctx.start)); -} - -/** Build spider options from the request body fields. */ -function buildSpiderOptions( - b: Record<string, unknown>, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): SpiderOptions { - return { - fetchOptions, - ...(typeof b["maxPages"] === "number" ? { maxPages: b["maxPages"] } : {}), - ...(typeof b["maxDepth"] === "number" ? { maxDepth: b["maxDepth"] } : {}), - ...(typeof b["sameDomain"] === "boolean" ? { sameDomain: b["sameDomain"] } : {}), - ...(typeof b["pathPrefix"] === "string" ? { pathPrefix: b["pathPrefix"] } : {}), - ...(Array.isArray(b["excludePatterns"]) - ? { - excludePatterns: (b["excludePatterns"] as unknown[]).filter( - (p): p is string => typeof p === "string", - ), - } - : {}), - }; -} - -/** Validate that a body field is a positive integer (if present). Returns error message or null. */ -function validatePositiveInt( - b: Record<string, unknown>, - field: string, - label: string, -): string | null { - if (b[field] === undefined) return null; - const v = b[field]; - if (typeof v !== "number" || !Number.isFinite(v) || !Number.isInteger(v) || v < 1) { - return `${label} must be a positive integer`; - } - return null; -} - -/** Validate that a body field is a non-negative integer (if present). Returns error message or null. */ -function validateNonNegativeInt( - b: Record<string, unknown>, - field: string, - label: string, -): string | null { - if (b[field] === undefined) return null; - const v = b[field]; - if (typeof v !== "number" || !Number.isFinite(v) || !Number.isInteger(v) || v < 0) { - return `${label} must be a non-negative integer`; - } - return null; -} - -/** Handle spidering (crawling) a URL and indexing all pages. */ -async function handleSpiderUrl( - ctx: RouteContext, - b: Record<string, unknown>, - urlStr: string, - topicId: string | undefined, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): Promise<void> { - const maxPagesErr = validatePositiveInt(b, "maxPages", "maxPages"); - if (maxPagesErr) { - sendError(ctx.res, 400, "VALIDATION_ERROR", maxPagesErr); - return; - } - const maxDepthErr = validateNonNegativeInt(b, "maxDepth", "maxDepth"); - if (maxDepthErr) { - sendError(ctx.res, 400, "VALIDATION_ERROR", maxDepthErr); - return; - } - - const spiderOptions = buildSpiderOptions(b, fetchOptions); - const indexedDocs: Array<{ id: string; title: string; url: string }> = []; - const errors: Array<{ url: string; error: string }> = []; - let stats: SpiderStats = { pagesFetched: 0, pagesCrawled: 0, pagesSkipped: 0, errors }; +let spec: Record<string, unknown> | undefined; - const gen = spiderUrl(urlStr, spiderOptions); - let result = await gen.next(); - while (!result.done) { - const page = result.value; - try { - const doc = await indexDocument(ctx.db, ctx.provider, { - content: page.content, - title: page.title, - sourceType: "manual", - url: page.url, - topicId, - }); - indexedDocs.push({ id: doc.id, title: page.title, url: page.url }); - } catch (indexErr) { - const msg = indexErr instanceof Error ? indexErr.message : String(indexErr); - errors.push({ url: page.url, error: msg }); - } - result = await gen.next(); - } - // result.value is SpiderStats when done (generator is exhausted) - if (result.done && result.value) { - stats = result.value; - stats.errors = errors; - } - - sendJson( - ctx.res, - 201, - { - documents: indexedDocs, - pagesFetched: indexedDocs.length, - pagesCrawled: stats.pagesCrawled, - pagesSkipped: stats.pagesSkipped, - errors, - abortReason: stats.abortReason ?? null, +const FIXED_ROUTES: ApiRoute[] = [ + { + method: "GET", + path: OPENAPI_PATH, + summary: "This OpenAPI document", + respond: (res): void => { + spec ??= buildOpenApiSpec(API_ROUTES); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify(spec)); }, - elapsed(ctx.start), - ); -} - -async function handleIndexFromUrl(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (typeof b["url"] !== "string") { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'url' is required"); - return; - } - const urlStr = b["url"]; - const topicId = typeof b["topic"] === "string" ? b["topic"] : undefined; - const config = loadConfig(); - const fetchOptions = { - allowPrivateUrls: config.indexing.allowPrivateUrls, - allowSelfSignedCerts: config.indexing.allowSelfSignedCerts, - }; - - // Spider mode — crawl linked pages - if (b["spider"] === true) { - await handleSpiderUrl(ctx, b, urlStr, topicId, fetchOptions); - return; - } - - // Single-URL mode (default) - const fetched = await fetchAndConvert(urlStr, fetchOptions); - const doc = await indexDocument(ctx.db, ctx.provider, { - content: fetched.content, - title: fetched.title, - sourceType: "manual", - url: urlStr, - topicId, - }); - sendJson(ctx.res, 201, doc, elapsed(ctx.start)); -} - -/** Handle SSE streaming for ask endpoint. */ -async function handleAskStream(ctx: RouteContext, b: Record<string, unknown>): Promise<void> { - const config = loadConfig(); - const llm = createLlmProvider(config); - const topic = typeof b["topic"] === "string" ? b["topic"] : undefined; - - ctx.res.writeHead(200, { - "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", - Connection: "keep-alive", - }); - - let clientDisconnected = false; - const disconnectPromise = new Promise<void>((resolve) => { - ctx.req.on("close", () => { - clientDisconnected = true; - resolve(); - }); - }); - - try { - const stream = askQuestionStream(ctx.db, ctx.provider, llm, { - question: b["question"] as string, - topic, - }); - - for await (const event of stream) { - if (clientDisconnected) break; - const ok = ctx.res.write(`data: ${JSON.stringify(event)}\n\n`); - if (!ok) { - await Promise.race([ - new Promise<void>((resolve) => ctx.res.once("drain", resolve)), - disconnectPromise, - ]); - } - } - } catch (streamErr: unknown) { - const message = streamErr instanceof Error ? streamErr.message : "Internal server error"; - ctx.res.write(`data: ${JSON.stringify({ error: message })}\n\n`); - } - - ctx.res.end(); -} - -async function handleAsk(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (typeof b["question"] !== "string") { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'question' is required"); - return; - } - - const accept = ctx.req.headers["accept"] ?? ""; - if (accept.includes("text/event-stream")) { - await handleAskStream(ctx, b); - return; - } - - const config = loadConfig(); - const llm = createLlmProvider(config); - const topic = typeof b["topic"] === "string" ? b["topic"] : undefined; - const result = await askQuestion(ctx.db, ctx.provider, llm, { question: b["question"], topic }); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); -} - -function handleListTopics(ctx: RouteContext): void { - const topics = listTopics(ctx.db); - sendJson(ctx.res, 200, topics, elapsed(ctx.start)); -} - -async function handleCreateTopic(ctx: RouteContext): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (typeof b["name"] !== "string") { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'name' is required"); - return; - } - const parentId = typeof b["parentId"] === "string" ? b["parentId"] : undefined; - const topic = createTopic(ctx.db, { name: b["name"], parentId }); - sendJson(ctx.res, 201, topic, elapsed(ctx.start)); -} - -function handleListTags(ctx: RouteContext): void { - const tags = listTags(ctx.db); - sendJson(ctx.res, 200, tags, elapsed(ctx.start)); -} - -async function handleAddTagsToDocument(ctx: RouteContext, docId: string): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - if (!Array.isArray(b["tags"])) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "Field 'tags' must be an array of strings"); - return; - } - const tagNames = (b["tags"] as unknown[]).filter((t): t is string => typeof t === "string"); - const tags = addTagsToDocument(ctx.db, docId, tagNames); - sendJson(ctx.res, 200, tags, elapsed(ctx.start)); -} - -function handleSuggestTags(ctx: RouteContext, docId: string): void { - const limitRaw = ctx.url.searchParams.get("limit"); - const limit = limitRaw ? Number.parseInt(limitRaw, 10) : 5; - const suggestions = suggestTags(ctx.db, docId, limit); - sendJson(ctx.res, 200, { documentId: docId, suggestions }, elapsed(ctx.start)); -} - -function handleSearchAnalytics(ctx: RouteContext): void { - const days = parseClampedInt(ctx.url, "days", 1, 365, 30); - const analytics = getSearchAnalytics(ctx.db, days); - const gaps = getKnowledgeGaps(ctx.db, days); - sendJson(ctx.res, 200, { ...analytics, knowledgeGaps: gaps }, elapsed(ctx.start)); -} - -function handleConnectorStatus(ctx: RouteContext): void { - const connectorType = ctx.url.searchParams.get("type") ?? undefined; - const connectorName = ctx.url.searchParams.get("name") ?? undefined; - const history = ctx.url.searchParams.get("history"); - const limit = parseClampedIntOrUndefined(ctx.url, "limit", 1, 1000); - - let data; - if (history === "true") { - data = getSyncHistory(ctx.db, connectorType, limit); - } else { - data = getConnectorStatus(ctx.db, connectorType, connectorName); - } - sendJson(ctx.res, 200, data, elapsed(ctx.start)); -} - -function handleScheduleStatus(ctx: RouteContext): void { - const entries = loadScheduleEntries(); - sendJson(ctx.res, 200, { schedules: entries }, elapsed(ctx.start)); -} - -function handleStats(ctx: RouteContext): void { - const config = loadConfig(); - const dbPath = config.database?.path; - const stats = getStats(ctx.db, dbPath); - sendJson(ctx.res, 200, stats, elapsed(ctx.start)); -} - -function handleGetDocument(ctx: RouteContext, docId: string): void { - const doc = getDocument(ctx.db, docId); - sendJson(ctx.res, 200, doc, elapsed(ctx.start)); -} - -function handleDeleteDocument(ctx: RouteContext, docId: string): void { - deleteDocument(ctx.db, docId); - ctx.res.writeHead(204); - ctx.res.end(); -} - -/** Parse and validate URL metadata for PATCH. Returns error message or null. */ -function validateUrlMetadata(urlStr: string): string | null { - try { - const parsedUrl = new URL(urlStr); - if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") { - return "URL must use http or https scheme"; - } - } catch { - return "Invalid URL format"; - } - return null; -} - -/** Extract an optional string field from the body, returning null for non-strings. */ -function extractOptionalString( - body: Record<string, unknown>, - key: string, -): string | null | undefined { - if (body[key] === undefined) return undefined; - return typeof body[key] === "string" ? body[key] : null; -} - -/** Parse metadata fields from an update request body. */ -function parseUpdateMetadata(body: Record<string, unknown>): - | { - library?: string | null; - version?: string | null; - url?: string | null; - topicId?: string | null; - } - | undefined { - const metadata: Record<string, string | null | undefined> = {}; - const library = extractOptionalString(body, "library"); - if (library !== undefined) metadata.library = library; - const version = extractOptionalString(body, "version"); - if (version !== undefined) metadata.version = version; - const url = extractOptionalString(body, "url"); - if (url !== undefined) metadata.url = url; - const topicId = extractOptionalString(body, "topicId"); - if (topicId !== undefined) metadata.topicId = topicId; - return Object.keys(metadata).length > 0 - ? (metadata as { - library?: string | null; - version?: string | null; - url?: string | null; - topicId?: string | null; - }) - : undefined; -} - -async function handleUpdateDocument(ctx: RouteContext, docId: string): Promise<void> { - const b = await requireJsonBody(ctx.req, ctx.res); - if (!b) return; - const title = typeof b["title"] === "string" ? b["title"] : undefined; - const content = typeof b["content"] === "string" ? b["content"] : undefined; - const metadata = parseUpdateMetadata(b); - if (metadata?.url) { - const urlErr = validateUrlMetadata(metadata.url); - if (urlErr) { - sendError(ctx.res, 400, "VALIDATION_ERROR", urlErr); - return; - } - } - const doc = await updateDocument(ctx.db, ctx.provider, docId, { title, content, metadata }); - sendJson(ctx.res, 200, doc, elapsed(ctx.start)); -} - -function handleGetDocumentLinks(ctx: RouteContext, docId: string): void { - const links = getDocumentLinks(ctx.db, docId); - sendJson(ctx.res, 200, links, elapsed(ctx.start)); -} - -async function handleCreateDocumentLink(ctx: RouteContext, docId: string): Promise<void> { - const body = (await parseJsonBody(ctx.req)) as { - targetId?: string; - linkType?: string; - label?: string; - }; - if (!body.targetId || !body.linkType) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "targetId and linkType are required"); - return; - } - const validLinkTypes = ["see_also", "prerequisite", "supersedes", "related"]; - if (!validLinkTypes.includes(body.linkType)) { - sendError( - ctx.res, - 400, - "VALIDATION_ERROR", - `Invalid linkType: ${body.linkType}. Must be one of: ${validLinkTypes.join(", ")}`, - ); - return; - } - const link = createLink(ctx.db, docId, body.targetId, body.linkType as LinkType, body.label); - sendJson(ctx.res, 201, link, elapsed(ctx.start)); -} - -function handleDeleteLink(ctx: RouteContext, linkId: string): void { - deleteLink(ctx.db, linkId); - ctx.res.writeHead(204); - ctx.res.end(); -} - -async function handleRunSavedSearch(ctx: RouteContext, searchId: string): Promise<void> { - const { search, results } = await runSavedSearch(ctx.db, ctx.provider, searchId); - sendJson(ctx.res, 200, { search, resultCount: results.length, results }, elapsed(ctx.start)); -} - -function handleListSavedSearches(ctx: RouteContext): void { - const searchLimit = parseClampedInt(ctx.url, "limit", 1, 1000, 50); - const searchOffset = parseClampedInt(ctx.url, "offset", 0, Number.MAX_SAFE_INTEGER, 0); - const searches = listSavedSearches(ctx.db, searchLimit, searchOffset); - sendJson(ctx.res, 200, searches, elapsed(ctx.start)); -} - -async function handleCreateSavedSearch(ctx: RouteContext): Promise<void> { - const body = (await parseJsonBody(ctx.req)) as { - name?: string; - query?: string; - filters?: Record<string, unknown>; - }; - if (!body.name || !body.query) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "name and query are required"); - return; - } - const saved = createSavedSearch(ctx.db, body.name, body.query, body.filters); - sendJson(ctx.res, 201, saved, elapsed(ctx.start)); -} - -function handleDeleteSavedSearch(ctx: RouteContext, searchId: string): void { - deleteSavedSearch(ctx.db, searchId); - ctx.res.writeHead(204); - ctx.res.end(); -} - -async function handleBulkOperation(ctx: RouteContext, operation: string): Promise<void> { - const body = (await parseJsonBody(ctx.req)) as { - selector?: BulkSelector; - dryRun?: boolean; - addTags?: string[]; - removeTags?: string[]; - targetTopicId?: string; - }; - - if (!body.selector) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "selector is required"); - return; - } - - if (operation === "delete") { - const result = bulkDelete(ctx.db, body.selector, body.dryRun ?? false); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); - return; - } - - if (operation === "retag") { - const result = bulkRetag( - ctx.db, - body.selector, - body.addTags, - body.removeTags, - body.dryRun ?? false, - ); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); - return; - } - - if (operation === "move") { - if (!body.targetTopicId) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "targetTopicId is required"); - return; - } - const result = bulkMove(ctx.db, body.selector, body.targetTopicId, body.dryRun ?? false); - sendJson(ctx.res, 200, result, elapsed(ctx.start)); - return; - } - - sendError( - ctx.res, - 400, - "VALIDATION_ERROR", - `Invalid bulk operation: ${operation}. Valid operations are: delete, retag, move`, - ); -} - -function handleListWebhooks(ctx: RouteContext): void { - const webhookLimit = parseClampedInt(ctx.url, "limit", 1, 1000, 50); - const webhookOffset = parseClampedInt(ctx.url, "offset", 0, Number.MAX_SAFE_INTEGER, 0); - const webhooks = listWebhooks(ctx.db, webhookLimit, webhookOffset); - sendJson(ctx.res, 200, webhooks.map(redactWebhook), elapsed(ctx.start)); -} - -async function handleCreateWebhook(ctx: RouteContext): Promise<void> { - const body = (await parseJsonBody(ctx.req)) as { - url?: string; - events?: string[]; - secret?: string; - }; - if (!body.url || !body.events) { - sendError(ctx.res, 400, "VALIDATION_ERROR", "url and events are required"); - return; - } - const webhook = await createWebhook(ctx.db, body.url, body.events as WebhookEvent[], body.secret); - sendJson(ctx.res, 201, redactWebhook(webhook), elapsed(ctx.start)); -} - -async function handleTestWebhook(ctx: RouteContext, webhookId: string): Promise<void> { - const webhook = getWebhook(ctx.db, webhookId); - await validateWebhookUrlSsrf(webhook.url); - const body = buildPayload("document.created", { test: true, message: "Webhook test ping" }); - const headers: Record<string, string> = { "Content-Type": "application/json" }; - if (webhook.secret) { - headers["X-LibScope-Signature"] = signPayload(body, webhook.secret); - } - const resp = await fetch(webhook.url, { - method: "POST", - headers, - body, - redirect: "error", - signal: AbortSignal.timeout(5000), - }); - sendJson(ctx.res, 200, { status: resp.status, statusText: resp.statusText }, elapsed(ctx.start)); -} - -function handleDeleteWebhook(ctx: RouteContext, webhookId: string): void { - deleteWebhook(ctx.db, webhookId); - ctx.res.writeHead(204); - ctx.res.end(); -} - -// --------------------------------------------------------------------------- -// Repo indexing webhook handler -// --------------------------------------------------------------------------- - -/** Match /api/v1/index/jobs/:jobId */ -function matchIndexJobId(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "index" && - segments[3] === "jobs" - ) { - return segments[4] ?? null; - } - return null; -} - -function handleGetIndexJob(ctx: RouteContext, jobId: string): void { - const job = repoIndexJobs.get(jobId); - if (!job) { - sendError(ctx.res, 404, "NOT_FOUND", `Index job "${jobId}" not found`); - return; - } - sendJson(ctx.res, 200, job, elapsed(ctx.start)); -} - -/** Match /api/v1/index/repos/:repoSlug */ -function matchRepoSlugForIndex(segments: string[]): string | null { - if ( - segments.length === 5 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === "index" && - segments[3] === "repos" - ) { - return segments[4] ?? null; - } - return null; -} - -async function handleIndexRepo(ctx: RouteContext, repoSlug: string): Promise<void> { - const log = getLogger(); - - // Parse optional body — empty body is valid (full reindex) - let body: Record<string, unknown> = {}; - try { - const parsed = await parseJsonBody(ctx.req); - if (parsed && typeof parsed === "object") { - body = parsed as Record<string, unknown>; - } - } catch { - /* empty body or non-JSON — treat as full reindex request */ - } - - const branch = typeof body["branch"] === "string" ? body["branch"] : undefined; - const files = Array.isArray(body["files"]) - ? (body["files"] as unknown[]).filter((f): f is string => typeof f === "string") - : undefined; - - const config = loadReposConfig(); - const entry = config.repos[repoSlug]; - if (!entry) { - sendError(ctx.res, 404, "NOT_FOUND", `Repo "${repoSlug}" not found in repos config`); - return; - } - - const jobId = randomUUID(); - const job: IndexJob = { - jobId, - repoSlug, - status: "queued", - startedAt: new Date().toISOString(), - }; - repoIndexJobs.set(jobId, job); - - sendJson(ctx.res, 202, { jobId, status: "queued" }, elapsed(ctx.start)); - - // Fire-and-forget indexing job - const libscope = createRepoLibScope(repoSlug); - job.status = "running"; - const indexOpts: { branch?: string; files?: string[] } = {}; - if (branch !== undefined) indexOpts.branch = branch; - if (files !== undefined) indexOpts.files = files; - indexRepo(libscope, repoSlug, entry, indexOpts) - .then((stats) => { - job.status = "completed"; - job.completedAt = new Date().toISOString(); - job.stats = stats; - log.info({ jobId, repoSlug, stats }, "Repo indexing completed"); - }) - .catch((err: unknown) => { - job.status = "failed"; - job.completedAt = new Date().toISOString(); - job.error = err instanceof Error ? err.message : String(err); - log.error({ jobId, repoSlug, err }, "Repo indexing failed"); - }) - .finally(() => { - libscope.close(); - }); -} - -// --------------------------------------------------------------------------- -// Error classification helper -// --------------------------------------------------------------------------- - -function handleRouteError( - res: ServerResponse, - err: unknown, + }, + { + method: "GET", + path: `${API_PREFIX}/health`, + summary: "Liveness check (use GET /api/v1/overview for counts and index health)", + respond: (res, start): void => sendJson(res, 200, { status: "ok" }, elapsed(start)), + }, +]; + +/** Every REST route: the fixed routes, then one per operation with an `http` mapping. */ +export const API_ROUTES: readonly ApiRoute[] = [ + ...FIXED_ROUTES, + ...OPERATIONS.flatMap((op): ApiRoute[] => + op.http + ? [ + { + method: op.http.method, + path: `${API_PREFIX}${op.http.path}`, + summary: op.summary, + operation: op, + }, + ] + : [], + ), +]; + +interface CompiledRoute { + route: ApiRoute; + parts: string[]; + paramCount: number; +} + +const COMPILED: CompiledRoute[] = API_ROUTES.map((route) => { + const parts = route.path.split("/").filter(Boolean); + return { route, parts, paramCount: parts.filter((p) => p.startsWith(":")).length }; +}); + +/** Path parameters when `segments` match `parts`, else null. */ +function matchParts(parts: string[], segments: string[]): Record<string, string> | null { + if (parts.length !== segments.length) return null; + const params: Record<string, string> = {}; + for (const [i, part] of parts.entries()) { + const segment = segments[i] ?? ""; + if (part.startsWith(":")) params[part.slice(1)] = decodeSegment(segment); + else if (part !== segment) return null; + } + return params; +} + +/** The route for a request; literal segments win over parameters. */ +export function matchRoute( method: string, pathname: string, -): void { - const log = getLogger(); - log.error({ err, method, pathname }, "API request error"); - - if (err instanceof DocumentNotFoundError) { - sendError(res, 404, "NOT_FOUND", err.message); - return; - } - if (err instanceof FetchError) { - sendError(res, 502, "FETCH_ERROR", err.message); - return; - } - if (err instanceof LibScopeError && err.code === "VALIDATION_ERROR") { - sendError(res, 400, "VALIDATION_ERROR", err.message); - return; +): { route: ApiRoute; params: Record<string, string> } | null { + const segments = pathname.split("/").filter(Boolean); + let best: { route: ApiRoute; params: Record<string, string>; paramCount: number } | null = null; + for (const c of COMPILED) { + if (c.route.method !== method || (best && best.paramCount <= c.paramCount)) continue; + const params = matchParts(c.parts, segments); + if (params) best = { route: c.route, params, paramCount: c.paramCount }; } - if (err instanceof Error && err.message === "Invalid JSON body") { - sendError(res, 400, "INVALID_JSON", "Request body contains invalid JSON"); - return; - } - - log.error({ err }, "Unhandled error in request handler"); - sendError(res, 500, "INTERNAL_ERROR", "Internal server error"); + return best; } -// --------------------------------------------------------------------------- -// Path-based route matching helpers (for segment-matched routes) -// --------------------------------------------------------------------------- - -function isApiV1Path(segments: string[], resource: string): boolean { - return ( - segments.length === 3 && - segments[0] === "api" && - segments[1] === "v1" && - segments[2] === resource - ); -} - -function isBulkPath(segments: string[]): boolean { - return ( - segments.length === 4 && segments[0] === "api" && segments[1] === "v1" && segments[2] === "bulk" - ); -} - -// --------------------------------------------------------------------------- -// Segment-based route dispatcher (document/:id, links, searches, webhooks, bulk) -// --------------------------------------------------------------------------- - -/** Dispatch single-document CRUD routes (GET/DELETE/PATCH /documents/:id). */ -async function dispatchDocumentCrudRoutes( - ctx: RouteContext, - docId: string, - method: string, -): Promise<boolean> { - if (method === "GET") { - handleGetDocument(ctx, docId); - return true; - } - if (method === "DELETE") { - handleDeleteDocument(ctx, docId); - return true; - } - if (method === "PATCH") { - await handleUpdateDocument(ctx, docId); - return true; - } - return false; -} - -/** Dispatch document-related segment routes (tags, suggest, CRUD, links). */ -async function dispatchDocumentRoutes( - ctx: RouteContext, - segments: string[], - method: string, -): Promise<boolean> { - const tagDocId = matchDocumentTags(segments); - if (tagDocId && method === "POST") { - await handleAddTagsToDocument(ctx, tagDocId); - return true; - } - const suggestDocId = matchDocumentSuggestTags(segments); - if (suggestDocId && method === "GET") { - handleSuggestTags(ctx, suggestDocId); - return true; - } - const docId = matchDocumentId(segments); - if (docId) return dispatchDocumentCrudRoutes(ctx, docId, method); - const linksDocId = matchDocumentLinks(segments); - if (linksDocId && method === "GET") { - handleGetDocumentLinks(ctx, linksDocId); - return true; - } - if (linksDocId && method === "POST") { - await handleCreateDocumentLink(ctx, linksDocId); - return true; - } - const linkId = matchLinkId(segments); - if (linkId && method === "DELETE") { - handleDeleteLink(ctx, linkId); - return true; - } - return false; -} - -/** Dispatch webhook segment routes. */ -async function dispatchWebhookRoutes( - ctx: RouteContext, - segments: string[], - method: string, -): Promise<boolean> { - if (isApiV1Path(segments, "webhooks") && method === "GET") { - handleListWebhooks(ctx); - return true; - } - if (isApiV1Path(segments, "webhooks") && method === "POST") { - await handleCreateWebhook(ctx); - return true; - } - const webhookTestId = matchWebhookTest(segments); - if (webhookTestId && method === "POST") { - await handleTestWebhook(ctx, webhookTestId); - return true; - } - const webhookId = matchWebhookId(segments); - if (webhookId && method === "DELETE") { - handleDeleteWebhook(ctx, webhookId); - return true; - } - return false; -} - -/** Dispatch search, bulk, and webhook segment routes. */ -async function dispatchMiscSegmentRoutes( - ctx: RouteContext, - segments: string[], - method: string, -): Promise<boolean> { - const searchRunId = matchSearchRun(segments); - if (searchRunId && method === "POST") { - await handleRunSavedSearch(ctx, searchRunId); - return true; - } - if (isApiV1Path(segments, "searches") && method === "GET") { - handleListSavedSearches(ctx); - return true; - } - if (isApiV1Path(segments, "searches") && method === "POST") { - await handleCreateSavedSearch(ctx); - return true; - } - if (isBulkPath(segments) && method === "POST") { - const operation = segments[3] as string; - await handleBulkOperation(ctx, operation); - return true; - } - const savedSearchId = matchSearchId(segments); - if (savedSearchId && method === "DELETE") { - handleDeleteSavedSearch(ctx, savedSearchId); - return true; - } - const indexJobId = matchIndexJobId(segments); - if (indexJobId && method === "GET") { - handleGetIndexJob(ctx, indexJobId); - return true; - } - const repoSlug = matchRepoSlugForIndex(segments); - if (repoSlug && method === "POST") { - await handleIndexRepo(ctx, repoSlug); - return true; - } - return dispatchWebhookRoutes(ctx, segments, method); -} - -async function dispatchSegmentRoutes( - ctx: RouteContext, - segments: string[], - method: string, -): Promise<boolean> { - const docHandled = await dispatchDocumentRoutes(ctx, segments, method); - if (docHandled) return true; - return dispatchMiscSegmentRoutes(ctx, segments, method); -} - -// --------------------------------------------------------------------------- -// Pathname-based route dispatcher (simple /api/v1/... paths) -// --------------------------------------------------------------------------- - -/** Route table: maps "METHOD /path" → handler. */ -const PATHNAME_ROUTES: Record<string, (ctx: RouteContext) => void | Promise<void>> = { - "GET /openapi.json": handleOpenApiSpec, - "GET /api/v1/health": handleHealthCheck, - "GET /api/v1/search": handleSearch, - "POST /api/v1/batch-search": handleBatchSearch, - "GET /api/v1/documents": handleListDocuments, - "POST /api/v1/documents": handleCreateDocument, - "POST /api/v1/documents/url": handleIndexFromUrl, - "POST /api/v1/ask": handleAsk, - "GET /api/v1/topics": handleListTopics, - "POST /api/v1/topics": handleCreateTopic, - "GET /api/v1/tags": handleListTags, - "GET /api/v1/analytics/searches": handleSearchAnalytics, - "GET /api/v1/connectors/status": handleConnectorStatus, - "GET /api/v1/connectors/schedules": handleScheduleStatus, - "GET /api/v1/stats": handleStats, -}; - -async function dispatchPathnameRoutes( - ctx: RouteContext, - pathname: string, - method: string, -): Promise<boolean> { - const handler = PATHNAME_ROUTES[`${method} ${pathname}`]; - if (!handler) return false; - await handler(ctx); - return true; +/** Input for an operation route: query string (GET, DELETE) or JSON body, plus path params. */ +async function buildInput( + op: Operation, + req: IncomingMessage, + url: URL, + params: Record<string, string>, +): Promise<Record<string, unknown>> { + const fromRequest = + op.http?.method === "GET" || op.http?.method === "DELETE" + ? inputFromParams(op, url.searchParams) + : await readJsonObject(req); + return { ...fromRequest, ...inputFromParams(op, params) }; } -// --------------------------------------------------------------------------- -// Main entry point -// --------------------------------------------------------------------------- - +/** Handle one REST request (after rate limiting, CORS and auth). */ export async function handleRequest( req: IncomingMessage, res: ServerResponse, - db: Database.Database, - provider: EmbeddingProvider, + ctx: OperationContext, ): Promise<void> { const start = performance.now(); - const url = parseUrl(req); - const pathname = url.pathname; + const url = new URL(req.url ?? "/", "http://localhost"); const method = req.method ?? "GET"; - const segments = extractPathSegments(pathname); - - const ctx: RouteContext = { req, res, db, provider, url, start }; - try { - // Try simple pathname-based routes first - const pathnameHandled = await dispatchPathnameRoutes(ctx, pathname, method); - if (pathnameHandled) return; - - // Try segment-matched routes (document/:id, links, searches, webhooks, bulk) - const segmentHandled = await dispatchSegmentRoutes(ctx, segments, method); - if (segmentHandled) return; - - // Unknown route — don't leak method/pathname to prevent endpoint enumeration - sendError(res, 404, "NOT_FOUND", "Route not found"); - } catch (err: unknown) { - handleRouteError(res, err, method, pathname); + const match = matchRoute(method, url.pathname); + // Same answer for unknown paths and wrong methods, so endpoints cannot be enumerated. + if (!match) throw new HttpError(404, "NOT_FOUND", "Route not found"); + const { route, params } = match; + if (route.respond) { + route.respond(res, start); + return; + } + const op = route.operation; + if (!op) throw new HttpError(404, "NOT_FOUND", "Route not found"); + const input = await buildInput(op, req, url, params); + const result = await respondWithOperation(op, ctx, input, TASKS_PATH); + if (result.location) res.setHeader("Location", result.location); + sendJson(res, result.status, result.data, elapsed(start)); + } catch (err) { + sendRequestError(res, err, { method, pathname: url.pathname }); } } diff --git a/src/api/server.ts b/src/api/server.ts index 3b9649c5..13f0a8b0 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -1,10 +1,21 @@ import { createServer, type Server } from "node:http"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; +import type { AddressInfo } from "node:net"; +import type { Bootstrapped } from "../core/bootstrap.js"; +import { createOperationContext } from "../core/operations/index.js"; +import { ConnectorScheduler, loadScheduleEntries } from "../core/scheduler.js"; import { getLogger } from "../logger.js"; -import { corsMiddleware, checkRateLimit, checkApiKey } from "./middleware.js"; +import { + checkApiKey, + checkRateLimit, + corsMiddleware, + rejectForeignWrite, + sendError, + setSecurityHeaders, +} from "./middleware.js"; import { handleRequest } from "./routes.js"; -import { ConnectorScheduler, loadScheduleEntries } from "../core/scheduler.js"; + +/** What a server needs from bootstrap(). */ +export type ServerApp = Pick<Bootstrapped, "db" | "provider" | "config">; /** Close an HTTP server, returning a promise that resolves when done. */ function closeHttpServer(server: Server): Promise<void> { @@ -13,17 +24,42 @@ function closeHttpServer(server: Server): Promise<void> { }); } +/** Listen on `port`/`host`; resolves with the port actually bound (for port 0). */ +export function listen(server: Server, port: number, host: string): Promise<number> { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, () => { + server.off("error", reject); + resolve((server.address() as AddressInfo).port); + }); + }); +} + +/** Answer 429 when `ip` is over the rate limit. Returns true when the request was rejected. */ +export function rejectRateLimited( + ip: string | undefined, + res: Parameters<typeof sendError>[0], +): boolean { + if (checkRateLimit(ip ?? "unknown")) return false; + res.setHeader("Retry-After", "60"); + sendError(res, 429, "RATE_LIMITED", "Too many requests"); + return true; +} + export interface ApiServerOptions { port?: number | undefined; host?: string | undefined; - /** Allowed CORS origins. Defaults to ["http://localhost", "http://localhost:3000"]. */ + /** + * Browser origins allowed to call the API. Default: ["http://localhost", + * "http://localhost:3000"]. "*" lets any origin read (GET) but not write. + */ corsOrigins?: string[] | undefined; enableScheduler?: boolean | undefined; } +/** Start the REST API server (default http://localhost:3378). */ export async function startApiServer( - db: Database.Database, - provider: EmbeddingProvider, + app: ServerApp, options?: ApiServerOptions, ): Promise<{ close: () => Promise<void>; @@ -31,58 +67,39 @@ export async function startApiServer( scheduler?: ConnectorScheduler | undefined; }> { const log = getLogger(); - const port = options?.port ?? 3378; const host = options?.host ?? "localhost"; const corsOrigins = options?.corsOrigins ?? ["http://localhost", "http://localhost:3000"]; + const ctx = createOperationContext({ ...app, surface: "api" }); const server = createServer((req, res) => { - // Rate limiting - const ip = req.socket.remoteAddress ?? "unknown"; - if (!checkRateLimit(ip)) { - res.writeHead(429, { "Content-Type": "application/json", "Retry-After": "60" }); - res.end(JSON.stringify({ error: { code: "RATE_LIMITED", message: "Too many requests" } })); - return; - } - + setSecurityHeaders(res); + if (rejectRateLimited(req.socket.remoteAddress, res)) return; if (corsMiddleware(req, res, corsOrigins)) return; + if (rejectForeignWrite(req, res, corsOrigins)) return; if (!checkApiKey(req, res)) return; - handleRequest(req, res, db, provider).catch((err: unknown) => { - log.error({ err }, "Unhandled error in request handler"); - if (!res.headersSent) { - res.writeHead(500, { "Content-Type": "application/json" }); - res.end( - JSON.stringify({ error: { code: "INTERNAL_ERROR", message: "Internal server error" } }), - ); - } - }); + // handleRequest answers every error itself. + void handleRequest(req, res, ctx); }); - return new Promise((resolve, reject) => { - server.on("error", reject); - server.listen(port, host, () => { - log.info({ port, host }, "API server started"); + const port = await listen(server, options?.port ?? 3378, host); + log.info({ port, host }, "API server started"); - let scheduler: ConnectorScheduler | undefined; - if (options?.enableScheduler !== false) { - const entries = loadScheduleEntries(); - if (entries.length > 0) { - scheduler = new ConnectorScheduler(db, provider); - scheduler.start(entries); - log.info( - { scheduledJobs: entries.length }, - "Connector scheduler started with API server", - ); - } - } + let scheduler: ConnectorScheduler | undefined; + if (options?.enableScheduler !== false) { + const entries = loadScheduleEntries(); + if (entries.length > 0) { + scheduler = new ConnectorScheduler(app.db, app.provider); + scheduler.start(entries); + log.info({ scheduledJobs: entries.length }, "Connector scheduler started with API server"); + } + } - resolve({ - close: async () => { - await scheduler?.stop(); - await closeHttpServer(server); - }, - port, - scheduler, - }); - }); - }); + return { + close: async (): Promise<void> => { + await scheduler?.stop(); + await closeHttpServer(server); + }, + port, + scheduler, + }; } diff --git a/src/cli/commands/add.ts b/src/cli/commands/add.ts new file mode 100644 index 00000000..1bea84b8 --- /dev/null +++ b/src/cli/commands/add.ts @@ -0,0 +1,165 @@ +/** `libscope add`: files, directories, URLs (optionally crawled) and repositories. */ +import type { Command } from "commander"; +import { statSync } from "node:fs"; +import { addOperation } from "../../core/operations/index.js"; +import type { IngestResult } from "../../core/ingest.js"; +import { resolveTopicId } from "../../core/topics.js"; +import { FileWatcher } from "../../core/watcher.js"; +import type { SourceType } from "../../core/indexing.js"; +import { ValidationError } from "../../errors.js"; +import { mapSequential } from "../../utils/async.js"; +import { getContext, untilInterrupted } from "../context.js"; +import { + addDocumentOptions, + defined, + documentInput, + splitList, + type DocumentFlags, +} from "../options.js"; +import { call, plural, print } from "../run.js"; + +interface AddFlags extends DocumentFlags { + title?: string; + format?: string; + dedup?: string; + expires?: string; + include?: string; + exclude?: string; + watch?: boolean; + spider?: boolean; + maxPages?: number; + maxDepth?: number; + sameDomain: boolean; + pathPrefix?: string; + excludeUrls?: string; + branch?: string; + path?: string; + extensions?: string; + token?: string; + dryRun?: boolean; +} + +const list = (value: string | undefined): string[] | undefined => + value === undefined ? undefined : splitList(value); + +/** add-operation input for one source. */ +export function addInput(source: string, flags: AddFlags): Record<string, unknown> { + return defined({ + source, + ...documentInput(flags), + title: flags.title, + format: flags.format, + dedup: flags.dedup, + expiresAt: flags.expires, + include: list(flags.include), + exclude: list(flags.exclude), + spider: flags.spider, + maxPages: flags.maxPages, + maxDepth: flags.maxDepth, + sameDomain: flags.spider ? flags.sameDomain : undefined, + pathPrefix: flags.pathPrefix, + excludePatterns: list(flags.excludeUrls), + branch: flags.branch, + paths: list(flags.path), + extensions: list(flags.extensions), + token: flags.token, + dryRun: flags.dryRun, + }); +} + +function printResult(result: IngestResult): void { + if (result.planned) { + console.log(`Would add ${plural(result.planned.length, "item")}:`); + for (const source of result.planned) console.log(` ${source}`); + } + for (const doc of result.documents) { + console.log(`✓ ${doc.title} (${plural(doc.chunkCount, "chunk")}) ${doc.documentId}`); + if (doc.source) console.log(` ${doc.source}`); + } + for (const s of result.skipped) console.log(` skipped ${s.source}: ${s.reason}`); + for (const e of result.errors) console.log(`✗ ${e.source}: ${e.error}`); + if (result.crawl) { + const stop = result.crawl.abortReason ? ` (stopped: ${result.crawl.abortReason})` : ""; + console.log( + `Crawled ${plural(result.crawl.pagesCrawled, "page")}, skipped ${result.crawl.pagesSkipped}${stop}`, + ); + } + if (!result.planned && result.documents.length !== 1) { + const errors = result.errors.length > 0 ? `, ${plural(result.errors.length, "error")}` : ""; + console.log(`Added ${plural(result.documents.length, "document")}${errors}`); + } +} + +/** Keep re-indexing changed files in `directories` until Ctrl+C. */ +async function watchDirectories(directories: string[], flags: AddFlags): Promise<void> { + const ctx = getContext({ embeddings: true }); + const document = { + topicId: flags.topic === undefined ? undefined : resolveTopicId(ctx.db, flags.topic), + library: flags.library, + version: flags.libVersion, + sourceType: flags.sourceType as SourceType | undefined, + }; + const watchers = directories.map( + (directory) => + new FileWatcher(ctx.db, ctx.provider, { + directory, + document, + onIndex: (path): void => console.log(`✓ indexed ${path}`), + onRemove: (path): void => console.log(`✗ removed ${path}`), + onError: (err): void => console.error(`⚠ ${err.message}`), + }), + ); + for (const w of watchers) w.start(); + console.error(`Watching ${directories.join(", ")} for changes (Ctrl+C to stop)...`); + await untilInterrupted(); + for (const w of watchers) w.stop(); +} + +function isDirectory(path: string): boolean { + try { + return statSync(path).isDirectory(); + } catch { + return false; + } +} + +export function register(program: Command): void { + const cmd = program + .command("add <sources...>") + .description( + "Add files, directories, URLs or GitHub/GitLab repository URLs to the knowledge base", + ); + addDocumentOptions(cmd, { mode: "assign", limit: false }) + .option("--title <title>", "Title (one file or URL; default: detected)") + .option("--format <ext>", "Read files as this format, e.g. .pdf") + .option("--dedup <mode>", "Duplicate handling: skip, warn or force") + .option("--expires <time>", "ISO 8601 time after which `admin prune` deletes the documents") + .option("--include <globs>", "Directory: only files matching these globs (comma-separated)") + .option("--exclude <globs>", "Directory: skip files matching these globs (comma-separated)") + .option("--watch", "Directory: keep running and re-index files when they change") + .option("--spider", "URL: also crawl linked pages") + .option("--max-pages <n>", "Crawl: page limit (default 25, max 200)", Number) + .option("--max-depth <n>", "Crawl: link depth (default 2, max 5)", Number) + .option("--no-same-domain", "Crawl: also follow links to other domains") + .option("--path-prefix <path>", "Crawl: only follow links under this path") + .option("--exclude-urls <globs>", "Crawl: skip URLs matching these globs (comma-separated)") + .option("--branch <name>", "Repository: branch (default: from the URL, else main)") + .option("--path <dirs>", "Repository: only these subdirectories (comma-separated)") + .option("--extensions <exts>", "Repository: file extensions (default .md,.mdx,.txt,.rst)") + .option("--token <token>", "Repository: access token for a private repository") + .option("--dry-run", "List what would be added without adding it") + .action(async (sources: string[], flags: AddFlags) => { + const directories = sources.filter(isDirectory); + if (flags.watch && directories.length !== sources.length) { + throw new ValidationError("--watch works with directories only"); + } + const results: IngestResult[] = await mapSequential(sources, (source) => + call(addOperation, addInput(source, flags), { embeddings: !flags.dryRun }), + ); + print(results, (all) => all.forEach(printResult)); + if (results.every((r) => r.documents.length === 0 && r.errors.length > 0)) { + process.exitCode = 1; + } + if (flags.watch && !flags.dryRun) await watchDirectories(directories, flags); + }); +} diff --git a/src/cli/commands/admin.ts b/src/cli/commands/admin.ts new file mode 100644 index 00000000..37d423b3 --- /dev/null +++ b/src/cli/commands/admin.ts @@ -0,0 +1,174 @@ +/** `libscope admin reindex|dedupe|backup|restore|stats|prune`. */ +import type { Command } from "commander"; +import { + backupOperation, + dedupeOperation, + overviewOperation, + popularOperation, + pruneExpiredOperation, + reindexOperation, + restoreOperation, + searchAnalyticsOperation, + staleOperation, +} from "../../core/operations/index.js"; +import type { Overview } from "../../core/overview.js"; +import { describeEmbeddingIdentity } from "../../db/index-meta.js"; +import { confirmOrCancel } from "../confirm.js"; +import { defined } from "../options.js"; +import { call, plural, print, printList, run } from "../run.js"; + +/** Counts, topics, packs and index identity (shared with `doctor`). */ +export function printOverview(o: Overview): void { + const s = o.stats; + console.log( + `Documents: ${s.totalDocuments} | chunks: ${s.totalChunks} | topics: ${s.totalTopics} | ` + + `database: ${(s.databaseSizeBytes / 1024).toFixed(1)} KB`, + ); + console.log(`Searches: ${s.totalSearches} (avg ${s.avgLatencyMs} ms)`); + const stored = describeEmbeddingIdentity(o.index.stored); + console.log(`Index: ${o.index.vectorTableDimensions === undefined ? "none" : stored}`); + if (o.packs.length > 0) { + const packs = o.packs.map((p) => `${p.name} v${p.version}`).join(", "); + console.log(`Packs: ${packs}`); + } +} + +type Stats = { + overview: Overview; + popular: Awaited<ReturnType<typeof popularOperation.handler>>; + stale: Awaited<ReturnType<typeof staleOperation.handler>>; + searches: Awaited<ReturnType<typeof searchAnalyticsOperation.handler>>; +}; + +function printStats(r: Stats): void { + printOverview(r.overview); + console.log("\nMost returned documents:"); + printList(r.popular.items, " (no searches yet)", (d) => + console.log(` ${String(d.hitCount).padStart(5)} ${d.title} ${d.documentId}`), + ); + console.log("\nDocuments no search returned:"); + printList(r.stale.items, " (none)", (d) => + console.log(` ${d.updatedAt.slice(0, 10)} ${d.title} ${d.documentId}`), + ); + const q = r.searches; + console.log(`\nSearches: ${q.totalSearches} (avg ${q.avgResultCount} results)`); + for (const t of q.topQueries) console.log(` ${String(t.count).padStart(5)}x ${t.query}`); + if (q.zeroResultQueries.length > 0) { + console.log("\nQueries with no results:"); + for (const t of q.zeroResultQueries) { + console.log(` ${String(t.count).padStart(5)}x ${t.query}`); + } + } +} + +function registerMaintenance(admin: Command): void { + admin + .command("reindex") + .description("Re-embed chunks with the configured embedding model") + .option("--rebuild", "Recreate the vector index for the configured model, then re-embed all") + .option("--doc <documentIds...>", "Only these documents") + .option("--since <date>", "Only documents created on or after (ISO 8601)") + .option("--before <date>", "Only documents created on or before (ISO 8601)") + .option("--batch-size <n>", "Chunks per embedding call (default 50)", Number) + .action( + async (flags: { + rebuild?: boolean; + doc?: string[]; + since?: string; + before?: string; + batchSize?: number; + }) => { + const input = defined({ + rebuild: flags.rebuild, + documentIds: flags.doc, + since: flags.since, + before: flags.before, + batchSize: flags.batchSize, + }); + await run( + reindexOperation, + input, + (r) => { + if (r.rebuiltIndex) console.log(`✓ Rebuilt vector index (${r.rebuiltIndex})`); + console.log(`✓ Re-embedded ${r.completed} of ${plural(r.total, "chunk")}`); + for (const id of r.failedChunkIds) console.log(`✗ failed: chunk ${id}`); + }, + // The operation creates or rebuilds the vector table itself. + { embeddings: true, vectorTable: "skip" }, + ); + }, + ); + + admin + .command("dedupe") + .description("Find duplicate and near-duplicate documents") + .option("--threshold <n>", "Similarity threshold 0-1 (default 0.95)", Number) + .option("--strategy <strategy>", "exact, semantic or both (default)") + .action(async (flags: { threshold?: number; strategy?: string }) => { + await run( + dedupeOperation, + defined({ ...flags }), + (r) => { + let group = 0; + printList(r.items, "No duplicates.", (g) => { + console.log(`Group ${++group} (${g.matchType}):`); + g.documentIds.forEach((id, i) => console.log(` ${id} ${g.titles[i] ?? ""}`)); + }); + }, + { embeddings: true }, + ); + }); + + admin + .command("prune") + .description("Delete documents whose expiry time has passed") + .action(async () => { + await run(pruneExpiredOperation, {}, (r) => + console.log(`✓ Pruned ${plural(r.pruned, "expired document")}`), + ); + }); +} + +function registerBackup(admin: Command): void { + admin + .command("backup <file>") + .description("Write the whole knowledge base to a JSON file") + .action(async (file: string) => { + await run(backupOperation, { outputPath: file }, (r) => + console.log(`✓ Backed up ${plural(r.counts.documents, "document")} to ${r.path}`), + ); + }); + + admin + .command("restore <file>") + .description("Import a file written by `admin backup`") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (file: string, flags: { yes?: boolean }) => { + const question = `Import ${file} into the current knowledge base?`; + if (!(await confirmOrCancel(question, flags.yes))) return; + await run(restoreOperation, { backupPath: file }, (r) => + console.log(`✓ Restored ${plural(r.counts.documents, "document")} from ${r.path}`), + ); + }); +} + +export function register(program: Command): void { + const admin = program.command("admin").description("Maintenance: index, backups, statistics"); + registerMaintenance(admin); + registerBackup(admin); + + admin + .command("stats") + .description("Counts, index, most returned and stale documents, and search analytics") + .option("--days <n>", "Look-back days for stale documents and search analytics", Number) + .action(async (flags: { days?: number }) => { + const days = defined({ days: flags.days }); + const stats: Stats = { + overview: await call(overviewOperation, {}), + popular: await call(popularOperation, {}), + stale: await call(staleOperation, days), + searches: await call(searchAnalyticsOperation, days), + }; + print(stats, printStats); + }); +} diff --git a/src/cli/commands/bulk.ts b/src/cli/commands/bulk.ts new file mode 100644 index 00000000..dd48fb11 --- /dev/null +++ b/src/cli/commands/bulk.ts @@ -0,0 +1,94 @@ +/** `libscope bulk delete|retag|move`: change every document that matches the filters. */ +import type { Command } from "commander"; +import type { z } from "zod"; +import { + bulkDeleteOperation, + bulkMoveOperation, + bulkRetagOperation, + type Operation, +} from "../../core/operations/index.js"; +import type { BulkResult } from "../../core/bulk.js"; +import { confirmOrCancel } from "../confirm.js"; +import { + addDocumentOptions, + defined, + documentInput, + splitList, + type DocumentFlags, +} from "../options.js"; +import { call, plural, print } from "../run.js"; + +interface BulkFlags extends DocumentFlags { + since?: string; + before?: string; + dryRun?: boolean; + yes?: boolean; +} + +function selectorInput(flags: BulkFlags): Record<string, unknown> { + return { ...documentInput(flags), ...defined({ dateFrom: flags.since, dateTo: flags.before }) }; +} + +function bulkCommand(parent: Command, name: string, description: string): Command { + return addDocumentOptions(parent.command(name).description(description), { + limit: false, + version: false, + }) + .option("--since <date>", "Only documents created on or after (ISO 8601)") + .option("--before <date>", "Only documents created on or before (ISO 8601)") + .option("--dry-run", "List the matching documents without changing them") + .option("-y, --yes", "Do not ask for confirmation"); +} + +/** Preview the matches, confirm, then apply (the operation itself refuses empty filters). */ +async function runBulk<S extends z.ZodObject>( + op: Operation<S, BulkResult>, + input: Record<string, unknown>, + flags: BulkFlags, + verb: string, +): Promise<void> { + const preview = await call(op, { ...input, dryRun: true }); + if (flags.dryRun || preview.affected === 0) { + print(preview, (r) => { + console.log(`${plural(r.affected, "document")} match:`); + for (const id of r.documentIds) console.log(` ${id}`); + }); + return; + } + if (!(await confirmOrCancel(`${verb} ${plural(preview.affected, "document")}?`, flags.yes))) { + return; + } + const result = await call(op, input); + print(result, (r) => console.log(`✓ ${verb}: ${plural(r.affected, "document")}`)); +} + +export function register(program: Command): void { + const bulk = program + .command("bulk") + .description("Change every document that matches the filters (at least one filter required)"); + + bulkCommand(bulk, "delete", "Delete matching documents").action(async (flags: BulkFlags) => { + await runBulk(bulkDeleteOperation, selectorInput(flags), flags, "Delete"); + }); + + bulkCommand(bulk, "retag", "Add and/or remove tags on matching documents") + .option("--add <tags>", "Tags to add (comma-separated)") + .option("--remove <tags>", "Tags to remove (comma-separated)") + .action(async (flags: BulkFlags & { add?: string; remove?: string }) => { + const input = { + ...selectorInput(flags), + ...defined({ + addTags: flags.add === undefined ? undefined : splitList(flags.add), + removeTags: flags.remove === undefined ? undefined : splitList(flags.remove), + }), + }; + await runBulk(bulkRetagOperation, input, flags, "Retag"); + }); + + bulkCommand(bulk, "move", "Move matching documents to another topic") + .requiredOption("--to <topic>", "Destination topic (ID or name)") + .action(async (flags: BulkFlags & { to: string }) => { + const input = { ...selectorInput(flags), targetTopic: flags.to }; + await runBulk(bulkMoveOperation, input, flags, `Move to "${flags.to}"`); + }); +} diff --git a/src/cli/commands/config.ts b/src/cli/commands/config.ts new file mode 100644 index 00000000..0e64e00a --- /dev/null +++ b/src/cli/commands/config.ts @@ -0,0 +1,66 @@ +/** `libscope config show|get|set|unset|path`. */ +import type { Command } from "commander"; +import { + CONFIG_KEY_NAMES, + getConfigFileFor, + getConfigValue, + getSecretsPath, + getUserConfigPath, + loadConfig, + maskConfigSecrets, + maskSecret, + setUserConfigValue, + unsetUserConfigValue, +} from "../../config.js"; +import { print } from "../run.js"; + +export function register(program: Command): void { + const config = program + .command("config") + .description(`Show and change settings (user file: ${getUserConfigPath()})`); + + config + .command("show") + .description("Show the effective configuration (API keys are masked)") + .action(() => { + console.log(JSON.stringify(maskConfigSecrets(loadConfig()), null, 2)); + }); + + config + .command("get <key>") + .description("Print the effective value of a key (API keys are masked)") + .action((key: string) => { + const value = getConfigValue(loadConfig(), key); + print({ key, value: value ?? null }, (r) => + console.log(r.value === null ? "" : String(r.value)), + ); + }); + + config + .command("set <key> <value>") + .description( + `Set a key in the user config file. Keys: ${CONFIG_KEY_NAMES.join(", ")}. ` + + `API keys are written to ${getSecretsPath()} (mode 0600), never to config.json.`, + ) + .action((key: string, value: string) => { + const stored = setUserConfigValue(key, value); + const file = getConfigFileFor(key); + const shown = file === getSecretsPath() ? maskSecret(String(stored)) : String(stored); + console.log(`✓ ${key} set to: ${shown} (${file})`); + }); + + config + .command("unset <key>") + .description("Remove a key from the user config file (or secrets file for API keys)") + .action((key: string) => { + const removed = unsetUserConfigValue(key); + console.log(removed ? `✓ ${key} removed` : `${key} is not set in ${getConfigFileFor(key)}`); + }); + + config + .command("path") + .description("Print the path of the user config file") + .action(() => { + console.log(getUserConfigPath()); + }); +} diff --git a/src/cli/commands/connectors.ts b/src/cli/commands/connectors.ts new file mode 100644 index 00000000..1f796062 --- /dev/null +++ b/src/cli/commands/connectors.ts @@ -0,0 +1,279 @@ +/** `libscope connect <type>`, `sync`, `disconnect`, `connections`. */ +import type { Command } from "commander"; +import { resolve } from "node:path"; +import { + disconnectOperation, + listConnectionsOperation, + syncOperation, +} from "../../core/operations/index.js"; +import { hasNamedConnectorConfig } from "../../connectors/index.js"; +import { getConnector, saveConnection } from "../../connectors/registry.js"; +import { + CONNECTOR_TYPES, + findSavedOneNoteConfig, + type ConnectorType, +} from "../../connectors/saved-config.js"; +import { authenticateDeviceCode } from "../../connectors/onenote.js"; +import { ValidationError } from "../../errors.js"; +import { confirmOrCancel } from "../confirm.js"; +import { defined, splitList } from "../options.js"; +import { plural, printList, run } from "../run.js"; + +export interface ConnectFlags { + name?: string; + schedule?: string; + sync: boolean; + token?: string; + exclude?: string; + channels?: string; + threadMode?: string; + email?: string; + server?: boolean; + spaces?: string; + topicMapping?: string; + notebook?: string; + clientId?: string; + tenantId?: string; + siteType?: string; + library?: string; + libVersion?: string; + maxPages?: number; + maxDepth?: number; + pathPrefix?: string; +} + +const list = (value: string | undefined): string[] | undefined => + value === undefined ? undefined : splitList(value); + +function requireSource(type: string, source: string | undefined, what: string): string { + if (!source) throw new ValidationError(`connect ${type} needs the ${what}`); + return source; +} + +/** Settings given on the command line for each connector type; undefined where not given. */ +const FLAG_SETTINGS: Record< + ConnectorType, + (source: string | undefined, flags: ConnectFlags) => Record<string, unknown> +> = { + notion: (_source, flags) => ({ token: flags.token, excludePages: list(flags.exclude) }), + slack: (_source, flags) => ({ + token: flags.token, + channels: list(flags.channels), + excludeChannels: list(flags.exclude), + threadMode: flags.threadMode, + }), + confluence: (source, flags) => ({ + baseUrl: source, + type: flags.server ? "server" : undefined, + email: flags.email, + token: flags.token, + spaces: list(flags.spaces), + excludeSpaces: list(flags.exclude), + }), + obsidian: (source, flags) => ({ + vaultPath: source && resolve(source), + topicMapping: flags.topicMapping, + excludePatterns: list(flags.exclude), + }), + onenote: (_source, flags) => ({ + clientId: flags.clientId, + tenantId: flags.tenantId, + notebooks: flags.notebook === undefined ? undefined : [flags.notebook], + excludeSections: undefined, + // A token given by hand cannot be refreshed. + ...(flags.token === undefined + ? {} + : { accessToken: flags.token, refreshToken: undefined, tokenExpiry: undefined }), + }), + docs: (source, flags) => ({ + url: source, + type: flags.siteType, + library: flags.library, + version: flags.libVersion, + maxPages: flags.maxPages, + maxDepth: flags.maxDepth, + pathPrefix: flags.pathPrefix, + }), +}; + +/** What the source argument is, for connector types that need it for a new connection. */ +const REQUIRED_SOURCE: Partial<Record<ConnectorType, string>> = { + confluence: "base URL", + obsidian: "vault path", + docs: "site URL", +}; + +/** Values a new connection gets for the settings that no flag sets (fresh objects each call). */ +const NEW_CONNECTION_DEFAULTS: Partial<Record<ConnectorType, () => Record<string, unknown>>> = { + slack: () => ({ channels: ["all"], threadMode: "aggregate" }), + confluence: () => ({ type: "cloud", spaces: ["all"] }), + obsidian: () => ({ topicMapping: "folder", excludePatterns: [] }), + onenote: () => ({ clientId: "", tenantId: "common", notebooks: ["all"], excludeSections: [] }), +}; + +/** `settings` of a new connection: checks the source and fills in the defaults. */ +function newConnectionSettings( + type: ConnectorType, + source: string | undefined, + settings: Record<string, unknown>, +): Record<string, unknown> { + const what = REQUIRED_SOURCE[type]; + if (what) requireSource(type, source, what); + const merged = { ...settings }; + for (const [key, value] of Object.entries(NEW_CONNECTION_DEFAULTS[type]?.() ?? {})) { + merged[key] ??= value; + } + return merged; +} + +/** + * Settings for connector `type` from the command line. Unset values are undefined, so the + * saved settings of the connection (if any) are kept; `defaults` apply to a new connection. + */ +export function connectSettings( + type: ConnectorType, + source: string | undefined, + flags: ConnectFlags, + isNew: boolean, +): Record<string, unknown> { + const settings = FLAG_SETTINGS[type](source, flags); + return isNew ? newConnectionSettings(type, source, settings) : settings; +} + +/** OneNote without a usable token: sign in with a device code (interactive). */ +async function oneNoteSignIn( + name: string, + settings: Record<string, unknown>, +): Promise<Record<string, unknown>> { + const saved = findSavedOneNoteConfig(name); + if (settings["accessToken"] !== undefined || saved?.refreshToken) return settings; + const clientId = (settings["clientId"] as string | undefined) ?? saved?.clientId; + if (!clientId) { + throw new ValidationError("OneNote sign-in needs --client-id (an Azure app client ID)"); + } + const tenantId = (settings["tenantId"] as string | undefined) ?? saved?.tenantId ?? "common"; + const auth = await authenticateDeviceCode(clientId, tenantId); + return { + ...settings, + clientId, + tenantId, + accessToken: auth.accessToken, + refreshToken: auth.refreshToken, + tokenExpiry: auth.expiresAt, + }; +} + +type SyncResult = Awaited<ReturnType<typeof syncOperation.handler>>; + +function printSync(result: SyncResult): void { + printList(result.items, "No saved connections. Add one with: libscope connect <type>", (item) => { + if (item.status === "failed") { + console.log(`✗ ${item.name}: ${item.error}`); + return; + } + const s = item.summary; + console.log( + `✓ ${item.name}: ${s.added} added, ${s.updated} updated, ${s.deleted} deleted, ${plural(s.errors.length, "error")}`, + ); + for (const e of s.errors) console.log(` ${e}`); + }); +} + +function registerConnect(program: Command): void { + program + .command("connect <type> [source]") + .summary("Save a connection to an external service and sync it") + .description( + `Save a connection and sync it. Types: ${CONNECTOR_TYPES.join(", ")}. ` + + "[source] is the vault path (obsidian) or the URL (confluence, docs). " + + "Running it again for a saved connection changes only the given settings.", + ) + .option("--name <name>", "Connection name (default: the type)") + .option( + "--schedule <cron>", + 'Sync on this cron schedule while `libscope serve api` runs ("off" removes it)', + ) + .option("--no-sync", "Save the connection without syncing now") + .option("--token <token>", "notion, slack, confluence: API token; onenote: access token") + .option( + "--exclude <list>", + "Comma-separated: slack channels, notion page IDs, confluence spaces, obsidian globs", + ) + .option("--channels <list>", "slack: channel names or IDs (default all)") + .option("--thread-mode <mode>", "slack: aggregate (one document per thread) or separate") + .option("--email <email>", "confluence: user email (Cloud)") + .option("--server", "confluence: Server/Data Center instead of Cloud") + .option("--spaces <keys>", "confluence: space keys (default all)") + .option("--topic-mapping <mode>", "obsidian: topics from folder (default) or frontmatter") + .option("--notebook <name>", "onenote: one notebook (default all)") + .option("--client-id <id>", "onenote: Azure app client ID for device sign-in") + .option("--tenant-id <id>", "onenote: Azure tenant ID (default common)") + .option("--site-type <type>", "docs: auto, sphinx, vitepress, doxygen or generic") + .option("--library <name>", "docs: library name for the pages") + .option("--lib-version <version>", "docs: library version for the pages") + .option("--max-pages <n>", "docs: page limit (default 500)", Number) + .option("--max-depth <n>", "docs: link depth (default 10)", Number) + .option("--path-prefix <path>", "docs: only pages under this path") + .action(async (type: string, source: string | undefined, flags: ConnectFlags) => { + const connector = getConnector(type); + const name = flags.name ?? connector.type; + const isNew = !hasNamedConnectorConfig(name); + let settings = connectSettings(connector.type, source, flags, isNew); + if (connector.type === "onenote") settings = await oneNoteSignIn(name, settings); + const schedule = flags.schedule === "off" ? null : flags.schedule; + saveConnection(name, connector.type, defined(settings), { schedule }); + console.error(`✓ Saved ${connector.label} connection "${name}"`); + if (!flags.sync) return; + await run(syncOperation, { name }, printSync, { embeddings: true }); + }); +} + +export function register(program: Command): void { + registerConnect(program); + + program + .command("sync [name]") + .description("Sync a saved connection with its saved settings, or all with --all") + .option("--all", "Sync every saved connection") + .action(async (name: string | undefined, flags: { all?: boolean }) => { + await run(syncOperation, defined({ name, all: flags.all }), printSync, { embeddings: true }); + }); + + program + .command("disconnect <name>") + .description("Delete a connection's documents and its saved settings (including credentials)") + .option("--type <type>", "Connector type, when no saved connection has this name") + .option("--keep-documents", "Only delete the saved settings") + .option("-y, --yes", "Do not ask for confirmation") + .action( + async (name: string, flags: { type?: string; keepDocuments?: boolean; yes?: boolean }) => { + const what = flags.keepDocuments ? "its saved settings" : "its documents and settings"; + if (!(await confirmOrCancel(`Disconnect "${name}" and delete ${what}?`, flags.yes))) { + return; + } + const input = defined({ name, type: flags.type, keepDocuments: flags.keepDocuments }); + await run(disconnectOperation, input, (r) => + console.log( + `✓ Disconnected ${r.type} "${r.name}": ${plural(r.documentsRemoved, "document")} removed`, + ), + ); + }, + ); + + program + .command("connections") + .description("List saved connections with their schedule and last sync") + .action(async () => { + await run(listConnectionsOperation, {}, (r) => + printList(r.items, "No saved connections. Add one with: libscope connect <type>", (c) => { + console.log(`${c.name} (${c.type})`); + if (c.schedule) console.log(` schedule: ${c.schedule}`); + if (c.lastSync) console.log(` last sync: ${c.lastSync}`); + if (c.lastRun) { + const error = c.lastRun.error ? ` — ${c.lastRun.error}` : ""; + console.log(` last run: ${c.lastRun.status} at ${c.lastRun.startedAt}${error}`); + } + }), + ); + }); +} diff --git a/src/cli/commands/docs.ts b/src/cli/commands/docs.ts new file mode 100644 index 00000000..1604b089 --- /dev/null +++ b/src/cli/commands/docs.ts @@ -0,0 +1,320 @@ +/** `libscope docs ...`: one document at a time (show, update, links, tags, ratings, versions). */ +import type { Command } from "commander"; +import { readFileSync } from "node:fs"; +import { + addTagsOperation, + deleteDocumentOperation, + documentHistoryOperation, + getDocumentOperation, + linkDocumentsOperation, + listDocumentsOperation, + listLinksOperation, + prerequisitesOperation, + rateDocumentOperation, + removeTagsOperation, + rollbackDocumentOperation, + suggestTagsOperation, + unlinkDocumentsOperation, + updateDocumentOperation, + type DocumentSummary, + type ListResult, +} from "../../core/operations/index.js"; +import type { DocumentView } from "../../core/document-view.js"; +import { LINK_TYPES } from "../../core/links.js"; +import { confirmOrCancel } from "../confirm.js"; +import { + addDocumentOptions, + defined, + documentInput, + splitList, + type DocumentFlags, +} from "../options.js"; +import { call, libraryLabel, plural, printList, run } from "../run.js"; + +/** Tags given as separate words and/or comma-separated. */ +function tagList(tags: string[]): string[] { + return tags.flatMap(splitList); +} + +function printDocumentList(result: ListResult<DocumentSummary>): void { + if (result.items.length === 0) { + console.log("No documents."); + return; + } + const last = result.offset + result.items.length; + console.log(`Documents ${result.offset + 1}-${last} of ${result.total}:\n`); + for (const d of result.items) { + console.log(`${d.documentId} ${d.title}`); + const lib = d.library ? ` | ${libraryLabel(d.library, d.version)}` : ""; + const url = d.url ? ` | ${d.url}` : ""; + console.log(` ${d.sourceType}${lib}${url} | updated ${d.updatedAt}`); + } +} + +function printDocument(view: DocumentView): void { + const d = view.document; + console.log(`# ${d.title}\n`); + console.log(`ID: ${d.documentId}`); + console.log(`Type: ${d.sourceType}`); + if (d.library) console.log(`Library: ${libraryLabel(d.library, d.version)}`); + if (d.topicId) console.log(`Topic: ${d.topicId}`); + if (d.url) console.log(`URL: ${d.url}`); + console.log(`Added by: ${d.submittedBy} (${d.createdAt}, updated ${d.updatedAt})`); + if (view.tags.length > 0) console.log(`Tags: ${view.tags.join(", ")}`); + const r = view.ratings; + if (r.totalRatings > 0) { + console.log( + `Rating: ${r.averageRating.toFixed(1)}/5 (${plural(r.totalRatings, "rating")}, ${plural(r.corrections, "correction")})`, + ); + } + for (const l of view.links.outgoing) { + console.log(`Link: → [${l.linkType}] ${l.targetTitle} (${l.targetId}) link ${l.id}`); + } + for (const l of view.links.incoming) { + console.log(`Link: ← [${l.linkType}] ${l.sourceTitle} (${l.sourceId}) link ${l.id}`); + } + console.log("\n---\n"); + console.log(view.content); + if (view.nextOffset !== null) { + console.log( + `\n... ${view.contentLength - view.nextOffset} more characters (--offset ${view.nextOffset})`, + ); + } +} + +interface UpdateFlags { + title?: string; + content?: string; + contentFile?: string; + library?: string; + libVersion?: string; + url?: string; + topic?: string; + tags?: string; +} + +export function updateInput(documentId: string, flags: UpdateFlags): Record<string, unknown> { + const content = + flags.contentFile === undefined ? flags.content : readFileSync(flags.contentFile, "utf-8"); + return defined({ + documentId, + title: flags.title, + content, + library: flags.library, + version: flags.libVersion, + url: flags.url, + topic: flags.topic, + tags: flags.tags === undefined ? undefined : splitList(flags.tags), + }); +} + +function registerViewCommands(docs: Command): void { + addDocumentOptions(docs.command("list").description("List documents, newest first")) + .option("--offset <n>", "Documents to skip (paging)", Number) + .action(async (flags: DocumentFlags & { offset?: number }) => { + await run( + listDocumentsOperation, + { ...documentInput(flags), ...defined({ offset: flags.offset }) }, + printDocumentList, + ); + }); + + docs + .command("show <documentId>") + .description("Show a document with its tags, links and ratings") + .option("--offset <n>", "Start the content at this character", Number) + .option("--max-length <n>", "Show at most this many characters of content", Number) + .action(async (documentId: string, flags: { offset?: number; maxLength?: number }) => { + await run( + getDocumentOperation, + defined({ documentId, offset: flags.offset, maxLength: flags.maxLength }), + printDocument, + ); + }); + + docs + .command("history <documentId>") + .description("List saved versions of a document") + .action(async (documentId: string) => { + await run(documentHistoryOperation, { documentId }, (r) => + printList(r.items, "No saved versions.", (v) => + console.log(`v${v.version} ${v.title} (${v.createdAt})`), + ), + ); + }); +} + +function registerEditCommands(docs: Command): void { + docs + .command("update <documentId>") + .description("Change a document's title, content, metadata or tags") + .option("--title <title>", "New title") + .option("--content <text>", "New content (re-chunked and re-embedded)") + .option("--content-file <path>", "Read the new content from a file") + .option("--library <name>", "New library name") + .option("--lib-version <version>", "New library version") + .option("--url <url>", "New URL") + .option("--topic <topic>", "New topic (ID or name)") + .option("--tags <tags>", "Replace the tags with these (comma-separated)") + .action(async (documentId: string, flags: UpdateFlags) => { + await run( + updateDocumentOperation, + updateInput(documentId, flags), + (view) => console.log(`✓ Updated "${view.document.title}" (${documentId})`), + { embeddings: true }, + ); + }); + + docs + .command("delete <documentId>") + .description("Delete a document with its chunks, links and ratings") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (documentId: string, flags: { yes?: boolean }) => { + const view = await call(getDocumentOperation, { documentId, maxLength: 0 }); + const title = view.document.title; + const question = `Delete "${title}" (${documentId})? This cannot be undone.`; + if (!(await confirmOrCancel(question, flags.yes))) return; + await run(deleteDocumentOperation, { documentId }, () => + console.log(`✓ Deleted "${title}" (${documentId})`), + ); + }); + + docs + .command("rollback <documentId> <version>") + .description("Restore a saved version (the current state is saved first)") + .action(async (documentId: string, version: string) => { + await run( + rollbackDocumentOperation, + { documentId, version: Number(version) }, + (v) => console.log(`✓ Restored version ${version}; the previous state is v${v.version}`), + { embeddings: true }, + ); + }); + + docs + .command("rate <documentId> <rating>") + .description("Rate a document from 1 (poor) to 5 (excellent)") + .option("--feedback <text>", "What is good or wrong") + .option("--correction <text>", "Suggested replacement text") + .option("--chunk <chunkId>", "Rate one chunk of the document") + .action( + async ( + documentId: string, + rating: string, + flags: { feedback?: string; correction?: string; chunk?: string }, + ) => { + const input = defined({ + documentId, + rating: Number(rating), + feedback: flags.feedback, + suggestedCorrection: flags.correction, + chunkId: flags.chunk, + }); + await run(rateDocumentOperation, input, (r) => + console.log(`✓ Rated ${documentId} ${r.rating}/5 rating ${r.id}`), + ); + }, + ); +} + +function registerLinkCommands(docs: Command): void { + docs + .command("link <documentId> <targetDocumentId>") + .description("Link one document to another") + .requiredOption("--type <type>", `Link type: ${LINK_TYPES.join(", ")}`) + .option("--label <text>", "Short description of the relationship") + .action( + async ( + documentId: string, + targetDocumentId: string, + flags: { type: string; label?: string }, + ) => { + const input = defined({ + documentId, + targetDocumentId, + linkType: flags.type, + label: flags.label, + }); + await run(linkDocumentsOperation, input, (l) => + console.log( + `✓ Linked ${documentId} -[${l.linkType}]-> ${targetDocumentId} link ${l.linkId}`, + ), + ); + }, + ); + + docs + .command("unlink <linkId>") + .description("Delete a link (link IDs are shown by `docs links`)") + .action(async (linkId: string) => { + await run(unlinkDocumentsOperation, { linkId }, () => + console.log(`✓ Deleted link ${linkId}`), + ); + }); + + docs + .command("links [documentId]") + .description("List the links of a document, or every link") + .option("--type <type>", `Only this link type: ${LINK_TYPES.join(", ")}`) + .action(async (documentId: string | undefined, flags: { type?: string }) => { + await run(listLinksOperation, defined({ documentId, linkType: flags.type }), (r) => + printList(r.items, "No links.", (l) => { + const dir = "direction" in l && l.direction === "incoming" ? "←" : "→"; + const label = l.label ? ` — ${l.label}` : ""; + console.log(`${l.linkId} ${l.sourceId} ${dir} [${l.linkType}] ${l.targetId}${label}`); + }), + ); + }); + + docs + .command("prereqs <documentId>") + .description("List the documents to read first (following prerequisite links)") + .action(async (documentId: string) => { + await run(prerequisitesOperation, { documentId }, (r) => { + let step = 0; + printList(r.items, "No prerequisites.", (d) => + console.log(`${++step}. ${d.title} ${d.documentId}`), + ); + }); + }); +} + +function registerTagCommands(docs: Command): void { + docs + .command("tag <documentId> <tags...>") + .description("Add tags to a document (space- or comma-separated)") + .action(async (documentId: string, tags: string[]) => { + await run(addTagsOperation, { documentId, tags: tagList(tags) }, (r) => + console.log(`✓ ${documentId} tags: ${r.tags.join(", ")}`), + ); + }); + + docs + .command("untag <documentId> <tags...>") + .description("Remove tags from a document") + .action(async (documentId: string, tags: string[]) => { + await run(removeTagsOperation, { documentId, tags: tagList(tags) }, (r) => + console.log( + `✓ Removed ${plural(r.removed.length, "tag")}; ${documentId} tags: ${r.tags.join(", ") || "(none)"}`, + ), + ); + }); + + docs + .command("suggest-tags <documentId>") + .description("Suggest tags from a document's content") + .option("-n, --limit <n>", "Maximum suggestions", Number) + .action(async (documentId: string, flags: { limit?: number }) => { + await run(suggestTagsOperation, defined({ documentId, limit: flags.limit }), (r) => + printList(r.suggestions, "No suggestions.", (t) => console.log(` ${t}`)), + ); + }); +} + +export function register(program: Command): void { + const docs = program.command("docs").description("Show and change documents"); + registerViewCommands(docs); + registerEditCommands(docs); + registerLinkCommands(docs); + registerTagCommands(docs); +} diff --git a/src/cli/commands/doctor.ts b/src/cli/commands/doctor.ts new file mode 100644 index 00000000..9f4e936f --- /dev/null +++ b/src/cli/commands/doctor.ts @@ -0,0 +1,193 @@ +/** + * `libscope doctor [--fix]`: config files, workspace and database, embedding model and index, + * LLM resolution, and health checks. With --fix it creates the database and vector index. + */ +import type { Command } from "commander"; +import { existsSync } from "node:fs"; +import { + getSecretsPath, + getUserConfigPath, + loadConfig, + resolveLlmProviderName, + type LibScopeConfig, +} from "../../config.js"; +import { overviewOperation } from "../../core/operations/index.js"; +import type { Overview } from "../../core/overview.js"; +import { getActiveWorkspace } from "../../core/workspace.js"; +import { resolveDatabasePath } from "../../db/connection.js"; +import { + describeEmbeddingIdentity, + toEmbeddingIdentity, + type EmbeddingIndexIdentity, +} from "../../db/index-meta.js"; +import { createEmbeddingProvider } from "../../providers/index.js"; +import { getGlobalOptions } from "../context.js"; +import { call, print } from "../run.js"; +import { printOverview } from "./admin.js"; + +export interface DoctorCheck { + name: string; + status: "ok" | "warn" | "error"; + detail: string; + /** What to run to fix a warning or error. */ + fix?: string | undefined; +} + +export interface DoctorReport { + configFile: { path: string; exists: boolean }; + secretsFile: { path: string; exists: boolean }; + workspace: string; + database: { path: string; exists: boolean }; + embedding: EmbeddingIndexIdentity; + llm: string; + overview?: Overview | undefined; + checks: DoctorCheck[]; +} + +const message = (err: unknown): string => (err instanceof Error ? err.message : String(err)); + +/** True when a field is known on both sides and differs. */ +function conflicts(a: EmbeddingIndexIdentity, b: EmbeddingIndexIdentity): boolean { + return (["provider", "model", "dimensions"] as const).some( + (k) => a[k] !== undefined && b[k] !== undefined && a[k] !== b[k], + ); +} + +function embeddingCheck(config: LibScopeConfig): { + identity: EmbeddingIndexIdentity; + check: DoctorCheck; +} { + try { + const identity = toEmbeddingIdentity(createEmbeddingProvider(config)); + return { + identity, + check: { name: "embedding", status: "ok", detail: describeEmbeddingIdentity(identity) }, + }; + } catch (err) { + return { + identity: { provider: config.embedding.provider }, + check: { + name: "embedding", + status: "error", + detail: message(err), + fix: "libscope config show", + }, + }; + } +} + +function llmCheck(config: LibScopeConfig): { llm: string; check: DoctorCheck } { + const name = resolveLlmProviderName(config, { surface: "cli" }); + if (name === null) { + return { + llm: "none", + check: { + name: "llm", + status: "warn", + detail: "No LLM configured: `ask` will not work (search does)", + fix: "libscope config set openai.apiKey <key> (or anthropic.apiKey, or llm.provider ollama)", + }, + }; + } + const model = name === "passthrough" ? "" : ` (${config.llm?.model ?? "default model"})`; + return { llm: name, check: { name: "llm", status: "ok", detail: `${name}${model}` } }; +} + +function indexChecks(overview: Overview, configured: EmbeddingIndexIdentity): DoctorCheck[] { + const checks: DoctorCheck[] = [ + { name: "database", status: overview.health.database, detail: "opens and is migrated" }, + { name: "keyword index", status: overview.health.fts, detail: "full-text index readable" }, + ]; + const stored: EmbeddingIndexIdentity = { + ...overview.index.stored, + dimensions: overview.index.vectorTableDimensions ?? overview.index.stored.dimensions, + }; + if (overview.index.vectorTableDimensions === undefined) { + checks.push({ + name: "vector index", + status: "warn", + detail: "not created yet (it is created when content is first added)", + fix: "libscope doctor --fix", + }); + } else if (conflicts(stored, configured)) { + checks.push({ + name: "vector index", + status: "error", + detail: `built with ${describeEmbeddingIdentity(stored)}, but ${describeEmbeddingIdentity(configured)} is configured`, + fix: "libscope admin reindex --rebuild", + }); + } else { + checks.push({ name: "vector index", status: "ok", detail: describeEmbeddingIdentity(stored) }); + } + return checks; +} + +export async function runDoctor(fix: boolean): Promise<DoctorReport> { + const config = loadConfig(); + const workspace = getGlobalOptions().workspace ?? getActiveWorkspace(); + const dbPath = resolveDatabasePath({ explicitPath: config.database.path, workspace }); + const embedding = embeddingCheck(config); + const llm = llmCheck(config); + const checks: DoctorCheck[] = [embedding.check, llm.check]; + let overview: Overview | undefined; + + if (existsSync(dbPath) || fix) { + try { + overview = await call(overviewOperation, {}, { vectorTable: fix ? "best-effort" : "skip" }); + checks.push(...indexChecks(overview, embedding.identity)); + } catch (err) { + checks.push({ + name: "database", + status: "error", + detail: message(err), + fix: "libscope admin reindex --rebuild", + }); + } + } else { + checks.push({ + name: "database", + status: "warn", + detail: "does not exist yet (it is created when content is first added)", + fix: "libscope doctor --fix", + }); + } + + return { + configFile: { path: getUserConfigPath(), exists: existsSync(getUserConfigPath()) }, + secretsFile: { path: getSecretsPath(), exists: existsSync(getSecretsPath()) }, + workspace, + database: { path: dbPath, exists: existsSync(dbPath) }, + embedding: embedding.identity, + llm: llm.llm, + overview, + checks, + }; +} + +const MARK = { ok: "✓", warn: "⚠", error: "✗" } as const; + +function printReport(r: DoctorReport): void { + const exists = (e: boolean): string => (e ? "" : " (not created)"); + console.log(`Config: ${r.configFile.path}${exists(r.configFile.exists)}`); + console.log(`Secrets: ${r.secretsFile.path}${exists(r.secretsFile.exists)}`); + console.log(`Workspace: ${r.workspace}`); + console.log(`Database: ${r.database.path}${exists(r.database.exists)}`); + if (r.overview) printOverview(r.overview); + console.log(""); + for (const c of r.checks) { + console.log(`${MARK[c.status]} ${c.name}: ${c.detail}`); + if (c.fix && c.status !== "ok") console.log(` fix: ${c.fix}`); + } +} + +export function register(program: Command): void { + program + .command("doctor") + .description("Check the setup: config, workspace, database, embedding model, index and LLM") + .option("--fix", "Create what is missing (database and vector index)") + .action(async (flags: { fix?: boolean }) => { + const report = await runDoctor(flags.fix === true); + print(report, printReport); + if (report.checks.some((c) => c.status === "error")) process.exitCode = 1; + }); +} diff --git a/src/cli/commands/pack.ts b/src/cli/commands/pack.ts new file mode 100644 index 00000000..63f6a37e --- /dev/null +++ b/src/cli/commands/pack.ts @@ -0,0 +1,159 @@ +/** `libscope pack install|remove|list|create`, over the pack operations. */ +import type { Command } from "commander"; +import { + createPackOperation, + installPackOperation, + listPacksOperation, + removePackOperation, +} from "../../core/operations/index.js"; +import type { InstallResult, InstalledPack } from "../../core/packs.js"; +import type { RegistryPack } from "../../registry/types.js"; +import { confirmOrCancel } from "../confirm.js"; +import { defined, splitList } from "../options.js"; +import { call, plural, print, printList, run } from "../run.js"; + +interface InstallFlags { + registry?: string; + batchSize?: number; + resumeFrom?: number; + concurrency?: number; +} + +function printInstall(result: InstallResult): void { + if (result.alreadyInstalled) { + console.log(`Pack "${result.packName}" is already installed.`); + return; + } + const errors = result.errors > 0 ? ` (${plural(result.errors, "error")})` : ""; + console.log( + `✓ Installed pack "${result.packName}": ${plural(result.documentsInstalled, "document")}${errors}`, + ); +} + +/** Print registry warnings (e.g. a registry that was never synced) to stderr. */ +export function printWarnings(warnings: readonly string[] | undefined): void { + for (const warning of warnings ?? []) console.error(`⚠ ${warning}`); +} + +function printPacks(result: { + items: Array<InstalledPack | RegistryPack>; + warnings?: string[] | undefined; +}): void { + printWarnings(result.warnings); + printList(result.items, "No packs found.", (p) => + console.log( + "installedAt" in p + ? `${p.name} v${p.version} — ${p.description ?? ""} (${plural(p.docCount, "document")})` + : `${p.name} v${p.latestVersion} — ${p.description} (registry ${p.registry})`, + ), + ); +} + +interface CreateFlags { + name: string; + from?: string[]; + topic?: string; + packVersion?: string; + description?: string; + author?: string; + license?: string; + output?: string; + extensions?: string; + exclude?: string[]; + recursive: boolean; +} + +export function createPackInput(flags: CreateFlags): Record<string, unknown> { + const fromSources = flags.from !== undefined && flags.from.length > 0; + return defined({ + name: flags.name, + from: fromSources ? flags.from : undefined, + topic: flags.topic, + version: flags.packVersion, + description: flags.description, + author: flags.author, + license: flags.license, + // Packs built from files can be large: gzip them by default. + outputPath: flags.output ?? `${flags.name}.json${fromSources ? ".gz" : ""}`, + extensions: flags.extensions === undefined ? undefined : splitList(flags.extensions), + exclude: flags.exclude, + recursive: flags.recursive, + }); +} + +export function register(program: Command): void { + const pack = program + .command("pack") + .description("Install, remove, list and create knowledge packs"); + + pack + .command("install <pack>") + .description( + "Install a pack from the configured registries (name or name@version) or a local .json/.json.gz file", + ) + .option("--registry <name>", "Look only in this registry") + .option("--batch-size <n>", "Documents embedded per batch (default 10)", Number) + .option("--resume-from <n>", "Skip the first N documents (resume a partial install)", Number) + .option("--concurrency <n>", "Batches embedded in parallel (default 4)", Number) + .action(async (name: string, flags: InstallFlags) => { + const input = defined({ + pack: name, + registry: flags.registry, + batchSize: flags.batchSize, + resumeFrom: flags.resumeFrom, + concurrency: flags.concurrency, + }); + await run(installPackOperation, input, printInstall, { embeddings: true }); + }); + + pack + .command("remove <name>") + .description("Remove an installed pack and its documents") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (name: string, flags: { yes?: boolean }) => { + const question = `Remove pack "${name}" and its documents? This cannot be undone.`; + if (!(await confirmOrCancel(question, flags.yes))) return; + await run(removePackOperation, { pack: name }, () => console.log(`✓ Removed pack "${name}"`)); + }); + + pack + .command("list") + .description("List installed packs, or the packs in the configured registries (--available)") + .option("--available", "List the packs in the configured registries") + .option("--registry <name>", "With --available: only this registry") + .action(async (flags: { available?: boolean; registry?: string }) => { + const input = defined({ available: flags.available, registry: flags.registry }); + await run(listPacksOperation, input, printPacks); + }); + + pack + .command("create") + .description("Create a pack from indexed documents, or from files, folders and URLs (--from)") + .requiredOption("--name <name>", "Pack name") + .option("--from <sources...>", "Files, folders or URLs to build the pack from") + .option("--topic <topic>", "Only this topic's documents (without --from)") + .option("--pack-version <version>", "Pack version (default 1.0.0)") + .option("--description <text>", "Pack description") + .option("--author <name>", "Pack author") + .option("--license <license>", "Pack license") + .option("--output <path>", "Output file (default <name>.json, or <name>.json.gz with --from)") + .option("--extensions <exts>", "With --from: file extensions to include (comma-separated)") + .option("--exclude <globs...>", "With --from: globs to skip") + .option("--no-recursive", "With --from: do not walk subdirectories") + .action(async (flags: CreateFlags) => { + const input = createPackInput(flags); + const created = await call(createPackOperation, input); + // The pack holds every document; print a summary instead of the whole pack. + const summary = { + name: created.name, + version: created.version, + documents: created.documents.length, + outputPath: input["outputPath"], + }; + print(summary, (s) => + console.log( + `✓ Created pack "${s.name}" with ${plural(s.documents, "document")} → ${String(s.outputPath)}`, + ), + ); + }); +} diff --git a/src/cli/commands/registry.ts b/src/cli/commands/registry.ts index cc7645b5..f51f80bb 100644 --- a/src/cli/commands/registry.ts +++ b/src/cli/commands/registry.ts @@ -1,51 +1,20 @@ -/** - * CLI commands for managing pack registries. - * Registered as `libscope registry <subcommand>`. - */ - +/** `libscope registry add|remove|list|sync|search|create|publish|unpublish`, over the registry operations. */ import type { Command } from "commander"; -import { rmSync } from "node:fs"; -import { resolve as pathResolve } from "node:path"; -import { - loadRegistries, - addRegistry, - removeRegistry, - validateRegistryName, - validateGitUrl, -} from "../../registry/config.js"; import { - cloneRegistry, - readIndex, - createRegistryRepo, - checkGitAvailable, -} from "../../registry/git.js"; -import { getRegistryCacheDir } from "../../registry/types.js"; -import { syncRegistryByName, syncAllRegistries } from "../../registry/sync.js"; -import { searchRegistries } from "../../registry/search.js"; -import { publishPack, publishPackToBranch, unpublishPack } from "../../registry/publish.js"; -import { confirmAction } from "../confirm.js"; - -/** Derive a short name from a git URL (e.g. "github.com/org/repo" → "repo"). */ -function deriveNameFromUrl(url: string): string { - // Try URL parsing first for https:// and ssh:// URLs - if (url.startsWith("https://") || url.startsWith("ssh://")) { - try { - const parsed = new URL(url); - const segments = parsed.pathname.split("/").filter(Boolean); - const last = segments[segments.length - 1] ?? ""; - const derived = last.replace(/\.git$/, ""); - if (derived) return derived; - } catch { - // fall through to SCP regex - } - } - - // Fall back to SCP-style SSH format: git@github.com:org/repo.git - const scpMatch = url.match(/:([^/]+\/)*([^/.]+?)(?:\.git)?$/); - if (scpMatch?.[2]) return scpMatch[2]; - - return "registry"; -} + addRegistryOperation, + createRegistryOperation, + listRegistriesOperation, + publishPackOperation, + removeRegistryOperation, + searchRegistriesOperation, + syncRegistriesOperation, + unpublishPackOperation, +} from "../../core/operations/index.js"; +import type { RegistrySearchResult, RegistrySyncStatus } from "../../registry/types.js"; +import { confirmOrCancel } from "../confirm.js"; +import { defined } from "../options.js"; +import { plural, printList, run } from "../run.js"; +import { printWarnings } from "./pack.js"; /** Truncate a string to a max length, adding "..." if truncated. */ function truncate(str: string, maxLen: number): string { @@ -59,376 +28,174 @@ function padColumns(cols: string[]): string { return cols.map((col, i) => col.padEnd(widths[i] ?? 16)).join(" "); } -/** Sync a single named registry and print the result. */ -async function syncSingleRegistry(name: string): Promise<void> { - const status = await syncRegistryByName(name); - if (status.status === "error") { - console.error(`Error: ${status.error}`); - process.exit(1); - return; - } - if (status.status === "offline") { - console.warn(`Warning: ${status.error}`); - console.warn( - `Registry "${name}" is unreachable. Using cached index from ${status.lastSyncedAt ?? "unknown"}.`, - ); - return; - } - const cacheDir = getRegistryCacheDir(name); - const index = readIndex(cacheDir); - console.log(`Registry "${name}" synced: ${index.length} pack(s) available.`); -} - -/** Sync all registries and print per-registry results. */ -async function syncAllRegistriesAction(): Promise<void> { - const results = await syncAllRegistries(); - if (results.length === 0) { - console.log("No registries configured."); - return; - } - for (const status of results) { - if (status.status === "success") { - const cacheDir = getRegistryCacheDir(status.registryName); - const index = readIndex(cacheDir); - console.log(` ${status.registryName}: synced (${index.length} packs)`); - } else if (status.status === "offline") { - console.warn(` ${status.registryName}: offline (using cached data)`); - } else { - console.error(` ${status.registryName}: error — ${status.error}`); - } - } +function packCount(packs: number | null): string { + return packs === null ? "not synced" : plural(packs, "pack"); } -/** Format an error for console output. */ -function formatError(err: unknown): string { - return err instanceof Error ? err.message : String(err); -} - -/** Require git to be available, exiting if not. */ -async function requireGit(): Promise<void> { - if (!(await checkGitAvailable())) { - console.error("Error: git is not installed or not in PATH."); - process.exit(1); - } -} - -/** Derive or validate the registry name from URL/option. Returns the name or exits. */ -function resolveRegistryName(url: string, explicitName?: string): string { - if (explicitName) return explicitName; - - const derived = deriveNameFromUrl(url); - try { - validateRegistryName(derived); - } catch { +/** One line per sync result; a registry that could not be synced sets exit code 1. */ +function printSyncStatus(status: RegistrySyncStatus): void { + if (status.status === "success") { + console.log(`✓ ${status.registry}: synced (${packCount(status.packs)})`); + } else if (status.status === "offline") { console.error( - `Error: Could not derive a valid registry name from URL (got "${derived}"). ` + - "Please provide a name using --name.", + `⚠ ${status.registry}: unreachable, using the local copy from ${status.lastSyncedAt ?? "unknown"} (${status.error ?? ""})`, ); - process.exit(1); + } else { + console.error(`✗ ${status.registry}: ${status.error ?? "sync failed"}`); + process.exitCode = 1; } - return derived; } -/** Parse and validate a non-negative integer option. Exits on invalid input. */ -function parseNonNegativeInt(value: string, optionName: string): number { - const n = Number.parseInt(value, 10); - if (Number.isNaN(n) || n < 0) { - console.error(`Error: "${optionName}" must be a non-negative integer.`); - process.exit(1); +function printSearchResults(result: { items: RegistrySearchResult[]; warnings: string[] }): void { + printWarnings(result.warnings); + if (result.items.length === 0) { + console.log("No packs found."); + return; } - return n; -} - -/** Perform the initial clone/sync after adding a registry. */ -async function initialSync(name: string, url: string): Promise<void> { - const cacheDir = getRegistryCacheDir(name); - console.log(`Cloning registry to ${cacheDir}...`); - try { - await cloneRegistry(url, cacheDir); - const index = readIndex(cacheDir); - console.log(`Synced: ${index.length} pack(s) available.`); - } catch (syncErr) { - console.warn( - `Warning: initial sync failed (${formatError(syncErr)}). ` + - 'You can retry with "libscope registry sync".', + const header = padColumns(["Pack", "Description", "Tags", "Version", "Registry"]); + console.log(header); + console.log("-".repeat(header.length)); + for (const r of result.items) { + const tags = r.tags.length > 0 ? r.tags.join(", ") : "-"; + console.log( + padColumns([ + r.name, + truncate(r.description, 40), + truncate(tags, 20), + r.latestVersion, + r.registry, + ]), ); } } -/** Register all `registry` subcommands on the given Commander program. */ -export function registerRegistryCommands(program: Command): void { - const registryCmd = program +export function register(program: Command): void { + const registry = program .command("registry") - .description("Manage pack registries (git-backed)"); + .description("Manage pack registries (git repositories of packs)"); - // --- registry add --- - registryCmd + registry .command("add <url>") - .description("Add a git-backed pack registry") - .option("-n, --name <alias>", "Short name for this registry") - .option("--priority <n>", "Priority for conflict resolution (lower wins, default: 10)", "10") - .option("--sync-interval <seconds>", "Auto-sync interval in seconds (0 = manual)", "0") - .option("--no-sync", "Skip the initial sync after adding") - .action( - async ( - url: string, - opts: { - name?: string; - priority: string; - syncInterval: string; - sync: boolean; + .description("Add a git repository as a pack registry and clone it") + .option("-n, --name <name>", "Registry name (default: from the URL)") + .option("--no-sync", "Do not clone the registry now") + .action(async (url: string, flags: { name?: string; sync: boolean }) => { + await run( + addRegistryOperation, + defined({ url, name: flags.name, sync: flags.sync }), + (result) => { + console.log(`✓ Registry "${result.name}" added (${result.url})`); + if (result.sync) printSyncStatus(result.sync); }, - ) => { - await requireGit(); - - const name = resolveRegistryName(url, opts.name); - const priority = parseNonNegativeInt(opts.priority, "--priority"); - const syncInterval = parseNonNegativeInt(opts.syncInterval, "--sync-interval"); - - try { - validateRegistryName(name); - validateGitUrl(url); - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - return; - } - - try { - addRegistry({ name, url, syncInterval, priority, lastSyncedAt: null }); - console.log(`Registry "${name}" added (${url}).`); - - if (opts.sync !== false) { - await initialSync(name, url); - } - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } - }, - ); + ); + }); - // --- registry remove --- - registryCmd + registry .command("remove <name>") - .description("Remove a registry and delete its local cache") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (name: string, opts: { yes?: boolean }) => { - if ( - !(await confirmAction( - `Remove registry "${name}" and its local cache? This cannot be undone.`, - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - - try { - removeRegistry(name); - - // Delete local cache - const cacheDir = getRegistryCacheDir(name); - try { - rmSync(cacheDir, { recursive: true, force: true }); - } catch (rmErr) { - console.warn(`Warning: could not remove cache directory (${formatError(rmErr)})`); - } - - console.log(`Registry "${name}" removed.`); - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } + .description("Remove a registry and delete its local clone") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (name: string, flags: { yes?: boolean }) => { + const question = `Remove registry "${name}" and its local clone?`; + if (!(await confirmOrCancel(question, flags.yes))) return; + await run(removeRegistryOperation, { name }, () => + console.log(`✓ Registry "${name}" removed`), + ); }); - // --- registry list --- - registryCmd + registry .command("list") - .description("List all configured registries") - .action(() => { - const registries = loadRegistries(); - if (registries.length === 0) { - console.log("No registries configured. Use 'libscope registry add <url>' to add one."); - return; - } - - console.log("Configured registries:\n"); - for (const reg of registries) { - const syncInfo = reg.lastSyncedAt ? `last synced ${reg.lastSyncedAt}` : "never synced"; - - // Try to read index to get pack count - let packCount = "?"; - try { - const cacheDir = getRegistryCacheDir(reg.name); - const index = readIndex(cacheDir); - packCount = String(index.length); - } catch { - // Cache doesn't exist yet — packCount remains "?" - } - - console.log( - ` ${reg.name} — ${reg.url} (priority: ${reg.priority}, ${packCount} packs, ${syncInfo})`, - ); - } + .description("List the configured registries") + .action(async () => { + await run(listRegistriesOperation, {}, (result) => + printList( + result.items, + "No registries configured. Add one with: libscope registry add <url>", + (r) => + console.log( + `${r.name} — ${r.url} (${packCount(r.packs)}, last synced ${r.lastSyncedAt ?? "never"})`, + ), + ), + ); }); - // --- registry sync --- - registryCmd + registry .command("sync [name]") - .description("Sync one or all registries (git fetch + fast-forward)") - .action(async (name?: string) => { - await requireGit(); - - if (name) { - await syncSingleRegistry(name); - } else { - await syncAllRegistriesAction(); - } + .description("Fetch the latest packs of one or all registries") + .action(async (name: string | undefined) => { + await run(syncRegistriesOperation, defined({ name }), (result) => + printList(result.items, "No registries configured.", printSyncStatus), + ); }); - // --- registry search --- - registryCmd + registry .command("search <query>") - .description("Search for packs across all configured registries") - .option("-r, --registry <name>", "Search only in a specific registry") - .action((query: string, opts: { registry?: string }) => { - const { results, warnings } = searchRegistries(query, { - registryName: opts.registry, - }); - - for (const w of warnings) { - console.warn(`Warning: ${w}`); - } - - if (results.length === 0) { - console.log(`No packs found matching "${query}".`); - return; - } - - console.log(`Found ${results.length} pack(s) matching "${query}":\n`); - - // Column header - const header = padColumns(["Pack", "Description", "Tags", "Version", "Registry"]); - console.log(header); - console.log("-".repeat(header.length)); - - for (const r of results) { - const tags = r.pack.tags.length > 0 ? r.pack.tags.join(", ") : "-"; - console.log( - padColumns([ - r.pack.name, - truncate(r.pack.description, 40), - truncate(tags, 20), - r.pack.latestVersion, - r.registryName, - ]), - ); - } + .description("Search the packs in the configured registries") + .option("--registry <name>", "Search only this registry") + .action(async (query: string, flags: { registry?: string }) => { + await run( + searchRegistriesOperation, + defined({ query, registry: flags.registry }), + printSearchResults, + ); }); - // --- registry create --- - registryCmd + registry .command("create <path>") - .description("Initialize a new registry git repo with canonical folder structure") - .action(async (rawPath: string) => { - await requireGit(); - - const resolved = pathResolve(rawPath); - try { - await createRegistryRepo(resolved); - console.log(`Registry repo initialized at ${resolved}`); - console.log("Push to a git remote, then add it with 'libscope registry add <url>'."); - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } + .description("Create an empty registry git repository") + .action(async (path: string) => { + await run(createRegistryOperation, { path }, (result) => { + console.log(`✓ Registry repository created at ${result.path}`); + console.log("Push it to a git remote, then add it with: libscope registry add <url>"); + }); }); - // --- registry publish --- - registryCmd - .command("publish <packFile>") - .description("Publish a pack file to a registry") - .requiredOption("-r, --registry <name>", "Target registry name") - .option("--version <semver>", "Version to publish as (default: auto-bump patch)") + registry + .command("publish <file>") + .description("Publish a pack file to a registry (commit and push)") + .requiredOption("--registry <name>", "Target registry") + .option("--pack-version <semver>", "Version to publish (default: next patch version)") .option("-m, --message <msg>", "Git commit message") - .option("--submit", "Push to a feature branch instead of main (for PR workflow)") + .option("--submit", "Push to a feature branch for a pull request instead of the main branch") .action( async ( - packFile: string, - opts: { registry: string; version?: string; message?: string; submit?: boolean }, + file: string, + flags: { registry: string; packVersion?: string; message?: string; submit?: boolean }, ) => { - await requireGit(); - - const resolved = pathResolve(packFile); - - try { - if (opts.submit) { - const result = await publishPackToBranch({ - registryName: opts.registry, - packFilePath: resolved, - version: opts.version, - commitMessage: opts.message, - }); + const input = defined({ + file, + registry: flags.registry, + version: flags.packVersion, + message: flags.message, + submit: flags.submit, + }); + await run(publishPackOperation, input, (result) => { + if (result.branch) { console.log( - `Pack "${result.packName}@${result.version}" pushed to branch "${result.branch}".`, + `✓ Pack "${result.packName}@${result.version}" pushed to branch "${result.branch}". Open a pull request to merge it.`, ); - console.log("Create a pull request to merge it into the registry."); } else { - const result = await publishPack({ - registryName: opts.registry, - packFilePath: resolved, - version: opts.version, - commitMessage: opts.message, - }); console.log( - `Pack "${result.packName}@${result.version}" published to "${result.registryName}" (checksum: ${result.checksum.slice(0, 12)}...).`, + `✓ Pack "${result.packName}@${result.version}" published to "${result.registryName}" (checksum ${result.checksum.slice(0, 12)}...)`, ); } - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } + }); }, ); - // --- registry unpublish --- - registryCmd - .command("unpublish <packName>") - .description("Remove a pack version from a registry") - .requiredOption("-r, --registry <name>", "Target registry name") - .requiredOption("--version <semver>", "Version to unpublish") + registry + .command("unpublish <pack>") + .description("Remove one version of a pack from a registry (<name>@<version>)") + .requiredOption("--registry <name>", "Target registry") .option("-m, --message <msg>", "Git commit message") - .option("-y, --yes", "Skip confirmation prompt") - .action( - async ( - packName: string, - opts: { registry: string; version: string; message?: string; yes?: boolean }, - ) => { - await requireGit(); - - if ( - !(await confirmAction( - `Unpublish "${packName}@${opts.version}" from "${opts.registry}"? This cannot be undone.`, - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - - try { - await unpublishPack({ - registryName: opts.registry, - packName, - version: opts.version, - commitMessage: opts.message, - }); - console.log(`Pack "${packName}@${opts.version}" unpublished from "${opts.registry}".`); - } catch (err) { - console.error(`Error: ${formatError(err)}`); - process.exit(1); - } - }, - ); + .option("-y, --yes", "Do not ask for confirmation") + .action(async (pack: string, flags: { registry: string; message?: string; yes?: boolean }) => { + const question = `Unpublish "${pack}" from "${flags.registry}"? This cannot be undone.`; + if (!(await confirmOrCancel(question, flags.yes))) return; + const input = defined({ pack, registry: flags.registry, message: flags.message }); + await run(unpublishPackOperation, input, (result) => + console.log( + `✓ Pack "${result.pack}@${result.version}" unpublished from "${result.registry}"`, + ), + ); + }); } diff --git a/src/cli/commands/search.ts b/src/cli/commands/search.ts new file mode 100644 index 00000000..ba7759d7 --- /dev/null +++ b/src/cli/commands/search.ts @@ -0,0 +1,130 @@ +/** `libscope search` (including interactive mode) and `libscope ask`. */ +import type { Command } from "commander"; +import type { ListResult } from "../../core/operations/index.js"; +import { askOperation, searchOperation } from "../../core/operations/index.js"; +import type { AnswerResult } from "../../core/rag.js"; +import type { SearchResult } from "../../core/search.js"; +import { ValidationError } from "../../errors.js"; +import { startInteractiveSearch } from "../repl.js"; +import { addDocumentOptions, defined, documentInput, type DocumentFlags } from "../options.js"; +import { isJsonOutput, libraryLabel, preview, run } from "../run.js"; + +/** One search hit: title, score, IDs, metadata and a content preview. */ +export function printSearchItem(r: SearchResult): void { + console.log(`\n── ${r.title} (score ${r.score.toFixed(2)})`); + console.log(` document ${r.documentId} chunk ${r.chunkId}`); + const meta = [r.library ? libraryLabel(r.library, r.version) : "", r.url ?? ""]; + const metaLine = meta.filter(Boolean).join(" | "); + if (metaLine) console.log(` ${metaLine}`); + for (const c of r.contextBefore ?? []) console.log(` ↑ ${preview(c.content, 120)}`); + console.log(` ${preview(r.content)}`); + for (const c of r.contextAfter ?? []) console.log(` ↓ ${preview(c.content, 120)}`); +} + +export function printSearchResults(result: ListResult<SearchResult>): void { + if (result.items.length === 0) { + console.log("No results."); + return; + } + const first = result.offset + 1; + const last = result.offset + result.items.length; + console.log(`Results ${first}-${last} of ${result.total}:`); + result.items.forEach(printSearchItem); +} + +function printAnswer(result: AnswerResult): void { + if (result.mode === "context") { + console.log("llm.provider is passthrough: no answer is generated. Context for the question:\n"); + console.log(result.contextPrompt); + } else { + console.log(`\n${result.answer}\n`); + } + if (result.sources.length > 0) { + console.log("Sources:"); + for (const s of result.sources) { + console.log(` • ${s.title} (score ${s.score.toFixed(2)}) ${s.documentId}`); + } + } + if (result.mode === "answer") { + const tokens = result.tokensUsed === undefined ? "" : ` | tokens: ${result.tokensUsed}`; + console.log(`\nModel: ${result.model}${tokens}`); + } +} + +interface SearchFlags extends DocumentFlags { + related?: string; + offset?: number; + minRating?: number; + maxPerDoc?: number; + context?: number; +} + +export function searchInput( + query: string | undefined, + flags: SearchFlags, +): Record<string, unknown> { + return defined({ + query, + relatedTo: flags.related, + ...documentInput(flags), + offset: flags.offset, + minRating: flags.minRating, + maxChunksPerDocument: flags.maxPerDoc, + contextChunks: flags.context, + }); +} + +interface AskFlags extends DocumentFlags { + minRating?: number; + model?: string; +} + +export function askInput(question: string, flags: AskFlags): Record<string, unknown> { + const { limit, ...filters } = documentInput(flags); + return defined({ question, ...filters, minRating: flags.minRating, topK: limit }); +} + +export function register(program: Command): void { + const search = program + .command("search [query]") + .description( + "Search by meaning and keywords. With no query on a terminal, start interactive search", + ) + .option("--related <id>", "Find content similar to this document or chunk ID instead"); + addDocumentOptions(search) + .option("--offset <n>", "Results to skip (paging)", Number) + .option("--min-rating <n>", "Only documents rated at least this (1-5)", Number) + .option("--max-per-doc <n>", "At most this many chunks per document", Number) + .option("--context <n>", "Neighbouring chunks to show around each result (0-2)", Number) + .action(async (query: string | undefined, flags: SearchFlags) => { + if (query === undefined && flags.related === undefined) { + if (!process.stdin.isTTY || isJsonOutput()) { + throw new ValidationError("Give a search query, or --related <documentId|chunkId>"); + } + await startInteractiveSearch({ + onQuery: async (q) => { + await run(searchOperation, searchInput(q, flags), printSearchResults, { + embeddings: true, + }); + }, + }); + return; + } + await run(searchOperation, searchInput(query, flags), printSearchResults, { + embeddings: true, + }); + }); + + const ask = program + .command("ask <question>") + .description("Answer a question from the knowledge base with the configured LLM"); + addDocumentOptions(ask, { limit: "Chunks to use as context (default 5)" }) + .option("--min-rating <n>", "Only documents rated at least this (1-5)", Number) + .option("--model <model>", "LLM model for this question (default: llm.model)"); + ask.action(async (question: string, flags: AskFlags) => { + await run(askOperation, askInput(question, flags), printAnswer, { + embeddings: true, + configOverrides: flags.model === undefined ? undefined : { llm: { model: flags.model } }, + }); + }); +} diff --git a/src/cli/commands/searches.ts b/src/cli/commands/searches.ts new file mode 100644 index 00000000..cde8359d --- /dev/null +++ b/src/cli/commands/searches.ts @@ -0,0 +1,71 @@ +/** `libscope searches ...`: saved searches. */ +import type { Command } from "commander"; +import { + deleteSavedSearchOperation, + listSavedSearchesOperation, + runSavedSearchOperation, + saveSearchOperation, +} from "../../core/operations/index.js"; +import { addDocumentOptions, defined, documentInput, type DocumentFlags } from "../options.js"; +import { printList, run } from "../run.js"; +import { printSearchItem } from "./search.js"; + +export function register(program: Command): void { + const searches = program.command("searches").description("Save and re-run searches"); + + addDocumentOptions( + searches.command("save <name> <query>").description("Save a query and its filters"), + ) + .option("--min-rating <n>", "Only documents rated at least this (1-5)", Number) + .action(async (name: string, query: string, flags: DocumentFlags & { minRating?: number }) => { + const input = { + name, + query, + ...documentInput(flags), + ...defined({ minRating: flags.minRating }), + }; + await run(saveSearchOperation, input, (s) => + console.log(`✓ Saved search "${s.name}" ${s.id}`), + ); + }); + + searches + .command("list") + .description("List saved searches") + .option("-n, --limit <n>", "Maximum results", Number) + .option("--offset <n>", "Results to skip (paging)", Number) + .action(async (flags: { limit?: number; offset?: number }) => { + await run(listSavedSearchesOperation, defined({ ...flags }), (r) => + printList(r.items, "No saved searches.", (s) => { + const filters = s.filters ? ` filters: ${JSON.stringify(s.filters)}` : ""; + console.log(`${s.name} ${s.id}`); + console.log(` query: "${s.query}"${filters}`); + if (s.lastRunAt) console.log(` last run ${s.lastRunAt} (${s.resultCount} results)`); + }), + ); + }); + + searches + .command("run <search>") + .description("Run a saved search (name or ID)") + .action(async (search: string) => { + await run( + runSavedSearchOperation, + { search }, + (r) => { + console.log(`Saved search "${r.search.name}": ${r.search.query}`); + printList(r.items, "No results.", printSearchItem); + }, + { embeddings: true }, + ); + }); + + searches + .command("delete <search>") + .description("Delete a saved search (name or ID)") + .action(async (search: string) => { + await run(deleteSavedSearchOperation, { search }, () => + console.log(`✓ Deleted saved search "${search}"`), + ); + }); +} diff --git a/src/cli/commands/serve.ts b/src/cli/commands/serve.ts new file mode 100644 index 00000000..be3a460c --- /dev/null +++ b/src/cli/commands/serve.ts @@ -0,0 +1,60 @@ +/** `libscope serve [mcp|api|dashboard]`. */ +import type { Command } from "commander"; +import { ValidationError } from "../../errors.js"; +import { + getContext, + getGlobalOptions, + leaveInterruptsToCommand, + untilInterrupted, +} from "../context.js"; + +const MODES = ["mcp", "api", "dashboard"] as const; + +async function serveMcp(): Promise<void> { + const { workspace, logLevel, verbose } = getGlobalOptions(); + const level = verbose ? "debug" : logLevel; + // stdout carries the MCP protocol: print nothing here. The server handles Ctrl+C itself. + leaveInterruptsToCommand(); + const mcp = await import("../../mcp/server.js"); + await mcp.runStdioServer({ + ...(workspace ? { workspace } : {}), + ...(level ? { logLevel: level } : {}), + }); +} + +export function register(program: Command): void { + program + .command("serve [mode]") + .description( + "Start the MCP server on stdio (mcp, default), the REST API (api, port 3378) or the web dashboard (dashboard, port 3377)", + ) + .option("--port <n>", "Port (api and dashboard)", Number) + .option("--host <host>", "Host to listen on (default localhost)") + .action(async (mode: string | undefined, flags: { port?: number; host?: string }) => { + const selected = mode ?? "mcp"; + if (!(MODES as readonly string[]).includes(selected)) { + throw new ValidationError(`Unknown serve mode "${selected}": use ${MODES.join(", ")}`); + } + if (selected === "mcp") { + await serveMcp(); + return; + } + const ctx = getContext({ embeddings: true }); + const app = { db: ctx.db, provider: ctx.provider, config: ctx.config }; + const host = flags.host ?? "localhost"; + if (selected === "api") { + const api = await import("../../api/server.js"); + const server = await api.startApiServer(app, { port: flags.port, host }); + console.error(`LibScope API listening on http://${host}:${server.port} (Ctrl+C to stop)`); + await untilInterrupted(); + await server.close(); + } else { + const web = await import("../../web/server.js"); + const port = flags.port ?? 3377; + const server = await web.startWebServer(app, { port, host }); + console.error(`LibScope dashboard at http://${host}:${port} (Ctrl+C to stop)`); + await untilInterrupted(); + server.close(); + } + }); +} diff --git a/src/cli/commands/topics.ts b/src/cli/commands/topics.ts new file mode 100644 index 00000000..44b490c2 --- /dev/null +++ b/src/cli/commands/topics.ts @@ -0,0 +1,71 @@ +/** `libscope topics ...` and `libscope tags list`. */ +import type { Command } from "commander"; +import { + createTopicOperation, + deleteTopicOperation, + listTagsOperation, + listTopicsOperation, +} from "../../core/operations/index.js"; +import { confirmOrCancel } from "../confirm.js"; +import { defined } from "../options.js"; +import { plural, printList, run } from "../run.js"; + +export function register(program: Command): void { + const topics = program.command("topics").description("Manage topics"); + + topics + .command("list") + .description("List topics with their document counts") + .option("--parent <topic>", "Only direct subtopics of this topic (ID or name)") + .action(async (flags: { parent?: string }) => { + await run(listTopicsOperation, defined({ parent: flags.parent }), (r) => + printList(r.items, "No topics. Create one with: libscope topics create <name>", (t) => + console.log(`${t.id} ${t.name} (${plural(t.documentCount, "document")})`), + ), + ); + }); + + topics + .command("create <name>") + .description("Create a topic") + .option("--description <text>", "What the topic covers") + .option("--parent <topic>", "Parent topic (ID or name)") + .action(async (name: string, flags: { description?: string; parent?: string }) => { + const input = defined({ name, description: flags.description, parent: flags.parent }); + await run(createTopicOperation, input, (t) => + console.log(`✓ Created topic ${t.name} ${t.id}`), + ); + }); + + topics + .command("delete <topic>") + .description("Delete a topic (its documents are kept without a topic)") + .option("--delete-documents", "Also delete the topic's documents") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (topic: string, flags: { deleteDocuments?: boolean; yes?: boolean }) => { + const what = flags.deleteDocuments + ? `topic "${topic}" and its documents` + : `topic "${topic}"`; + if (!(await confirmOrCancel(`Delete ${what}?`, flags.yes))) return; + await run( + deleteTopicOperation, + { topic, deleteDocuments: flags.deleteDocuments === true }, + (r) => console.log(`✓ Deleted topic ${r.topicId}`), + ); + }); + + program + .command("tags") + .description("List tags") + .command("list") + .description("List all tags with their document counts") + .action(async () => { + await run(listTagsOperation, {}, (r) => + printList( + r.items, + "No tags. Add some with: libscope docs tag <documentId> <tags...>", + (t) => console.log(`${t.name} (${plural(t.documentCount, "document")})`), + ), + ); + }); +} diff --git a/src/cli/commands/webhooks.ts b/src/cli/commands/webhooks.ts new file mode 100644 index 00000000..b499c4b4 --- /dev/null +++ b/src/cli/commands/webhooks.ts @@ -0,0 +1,59 @@ +/** `libscope webhooks list|create|delete|test`. */ +import type { Command } from "commander"; +import { + createWebhookOperation, + deleteWebhookOperation, + listWebhooksOperation, + testWebhookOperation, +} from "../../core/operations/index.js"; +import { WEBHOOK_EVENTS } from "../../core/webhooks.js"; +import { defined, splitList } from "../options.js"; +import { printList, run } from "../run.js"; + +export function register(program: Command): void { + const webhooks = program.command("webhooks").description("Send signed POSTs on document events"); + + webhooks + .command("list") + .description("List webhooks (secrets are never shown)") + .action(async () => { + await run(listWebhooksOperation, {}, (r) => + printList(r.items, "No webhooks.", (h) => { + console.log(`${h.id} ${h.url}`); + console.log(` events: ${h.events.join(", ")} | active: ${h.active ? "yes" : "no"}`); + if (h.lastTriggeredAt) console.log(` last triggered: ${h.lastTriggeredAt}`); + if (h.failureCount > 0) console.log(` failures: ${h.failureCount}`); + }), + ); + }); + + webhooks + .command("create <url>") + .description("Register a webhook") + .requiredOption("--events <events>", `Comma-separated: ${WEBHOOK_EVENTS.join(", ")}`) + .option("--secret <secret>", "HMAC-SHA256 signing secret (needs LIBSCOPE_SECRET_KEY)") + .action(async (url: string, flags: { events: string; secret?: string }) => { + const input = defined({ url, events: splitList(flags.events), secret: flags.secret }); + await run(createWebhookOperation, input, (h) => + console.log(`✓ Created webhook ${h.id} for ${h.events.join(", ")}`), + ); + }); + + webhooks + .command("delete <webhookId>") + .description("Delete a webhook") + .action(async (webhookId: string) => { + await run(deleteWebhookOperation, { webhookId }, () => + console.log(`✓ Deleted webhook ${webhookId}`), + ); + }); + + webhooks + .command("test <webhookId>") + .description("Send a test event and show the HTTP status") + .action(async (webhookId: string) => { + await run(testWebhookOperation, { webhookId }, (r) => + console.log(`Response: ${r.status} ${r.statusText}`), + ); + }); +} diff --git a/src/cli/commands/workspace.ts b/src/cli/commands/workspace.ts new file mode 100644 index 00000000..4d3b582d --- /dev/null +++ b/src/cli/commands/workspace.ts @@ -0,0 +1,55 @@ +/** `libscope workspace create|list|use|delete`. */ +import type { Command } from "commander"; +import { + createWorkspace, + deleteWorkspace, + getActiveWorkspace, + listWorkspaces, + setActiveWorkspace, +} from "../../core/workspace.js"; +import { confirmOrCancel } from "../confirm.js"; +import { print, printList } from "../run.js"; + +export function register(program: Command): void { + const workspace = program.command("workspace").description("Separate knowledge bases"); + + workspace + .command("create <name>") + .description("Create a workspace") + .action((name: string) => { + const ws = createWorkspace(name); + console.log(`✓ Created workspace "${ws.name}" at ${ws.path}`); + }); + + workspace + .command("list") + .description("List workspaces") + .action(() => { + const active = getActiveWorkspace(); + const items = listWorkspaces().map((ws) => ({ ...ws, active: ws.name === active })); + print({ items }, (r) => + printList(r.items, "No workspaces.", (ws) => + console.log(`${ws.active ? "* " : " "}${ws.name} ${ws.path}`), + ), + ); + }); + + workspace + .command("use <name>") + .description("Make a workspace the active one") + .action((name: string) => { + setActiveWorkspace(name); + console.log(`✓ Active workspace: ${name}`); + }); + + workspace + .command("delete <name>") + .description("Delete a workspace and its database") + .option("-y, --yes", "Do not ask for confirmation") + .action(async (name: string, flags: { yes?: boolean }) => { + const question = `Delete workspace "${name}" and its database? This cannot be undone.`; + if (!(await confirmOrCancel(question, flags.yes))) return; + deleteWorkspace(name); + console.log(`✓ Deleted workspace "${name}"`); + }); +} diff --git a/src/cli/confirm.ts b/src/cli/confirm.ts index 36871a1d..97862e17 100644 --- a/src/cli/confirm.ts +++ b/src/cli/confirm.ts @@ -22,3 +22,18 @@ export async function confirmAction( rl.close(); } } + +/** + * Like {@link confirmAction}, but prints "Cancelled." when the user declines. + * Returns true when the caller should proceed. + */ +export async function confirmOrCancel( + message: string, + yes: boolean | undefined, + /** Overridable for testing */ + createInterface?: () => readline.Interface, +): Promise<boolean> { + if (await confirmAction(message, !!yes, createInterface)) return true; + console.log("Cancelled."); + return false; +} diff --git a/src/cli/context.ts b/src/cli/context.ts new file mode 100644 index 00000000..accf20e5 --- /dev/null +++ b/src/cli/context.ts @@ -0,0 +1,140 @@ +/** + * Process-wide CLI state: the global options and the one operation context a command uses. + * The context is built once, on first use, with `bootstrap()`; commands that do not embed + * text open it without requiring the embedding provider, so they never load a model. + */ +import { bootstrap, type Bootstrapped, type ConfigOverrides } from "../core/bootstrap.js"; +import { createOperationContext, type OperationContext } from "../core/operations/index.js"; +import { LocalEmbeddingProvider, type ModelDownloadEvent } from "../providers/local.js"; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import type { LogLevel } from "../logger.js"; + +export interface GlobalOptions { + json?: boolean | undefined; + verbose?: boolean | undefined; + logLevel?: LogLevel | undefined; + workspace?: string | undefined; +} + +let globals: GlobalOptions = {}; + +export function setGlobalOptions(options: GlobalOptions): void { + globals = options; +} + +export function getGlobalOptions(): GlobalOptions { + return globals; +} + +export interface ContextOptions { + /** The command embeds text: the embedding provider must be usable. Default false. */ + embeddings?: boolean | undefined; + /** Vector table handling (default: "required" with embeddings, else "skip"). */ + vectorTable?: "required" | "best-effort" | "skip" | undefined; + /** Per-section config values merged over the loaded config. */ + configOverrides?: ConfigOverrides | undefined; +} + +interface OpenContext { + boot: Bootstrapped; + ctx: OperationContext; +} + +let open: OpenContext | undefined; +let controller = new AbortController(); + +/** Print one stderr line while the local model downloads (first run only). */ +function reportModelDownload(provider: EmbeddingProvider): void { + if (!(provider instanceof LocalEmbeddingProvider)) return; + const text = `Downloading embedding model ${provider.model} (first run only)...`; + const tty = process.stderr.isTTY; + let started = false; + provider.onDownloadProgress = (event: ModelDownloadEvent): void => { + if (!started) { + started = true; + process.stderr.write(tty ? text : `${text}\n`); + } + if (!tty) return; + if (event.status === "progress" && event.progress !== undefined) { + process.stderr.write(`\r${text} ${Math.round(event.progress)}% ${event.file ?? ""}`); + } else if (event.status === "ready") { + process.stderr.write(`\r${text} done\x1b[K\n`); + } + }; +} + +/** The operation context for this command, opened on first call. */ +export function getContext(options: ContextOptions = {}): OperationContext { + if (open) return open.ctx; + const embeddings = options.embeddings ?? false; + const boot = bootstrap({ + workspace: globals.workspace, + configOverrides: options.configOverrides, + requireProvider: embeddings, + vectorTable: options.vectorTable ?? (embeddings ? "required" : "skip"), + warn: (message) => process.stderr.write(`⚠ ${message}\n`), + }); + reportModelDownload(boot.provider); + const ctx = createOperationContext({ + db: boot.db, + provider: boot.provider, + config: boot.config, + surface: "cli", + signal: controller.signal, + }); + open = { boot, ctx }; + return ctx; +} + +/** Database file of the open context, if any. */ +export function getOpenDbPath(): string | undefined { + return open?.boot.dbPath; +} + +/** Close the database (safe to call when nothing is open). */ +export function closeContext(): void { + open?.boot.close(); + open = undefined; + controller = new AbortController(); +} + +/** Ask a running operation to stop (Ctrl+C). */ +export function abortContext(): void { + controller.abort(); +} + +/** True after abortContext() for the current command. */ +export function isAborted(): boolean { + return controller.signal.aborted; +} + +let waiters = 0; + +/** Resolves when the user presses Ctrl+C (or the process gets SIGTERM). For long-running commands. */ +export function untilInterrupted(): Promise<void> { + if (controller.signal.aborted) return Promise.resolve(); + const signal = controller.signal; + waiters++; + return new Promise((resolve) => + signal.addEventListener("abort", () => { + waiters--; + resolve(); + }), + ); +} + +/** True while a command waits in untilInterrupted() (it cleans up itself on Ctrl+C). */ +export function isWaitingForInterrupt(): boolean { + return waiters > 0; +} + +let commandHandlesInterrupts = false; + +/** The running command (the MCP server) handles Ctrl+C itself; the CLI must not exit. */ +export function leaveInterruptsToCommand(): void { + commandHandlesInterrupts = true; +} + +export function isInterruptLeftToCommand(): boolean { + return commandHandlesInterrupts; +} diff --git a/src/cli/errors.ts b/src/cli/errors.ts new file mode 100644 index 00000000..b90424a1 --- /dev/null +++ b/src/cli/errors.ts @@ -0,0 +1,41 @@ +/** The one CLI error handler: "✗ message", a next-step hint, and the stack only with --verbose. */ +import { ConfigError, NotFoundError, ValidationError } from "../errors.js"; + +const NOT_FOUND_HINTS: Record<string, string> = { + DOCUMENT_NOT_FOUND: "List document IDs with `libscope docs list`.", + CHUNK_NOT_FOUND: "Chunk IDs are shown in `libscope search` results.", + TOPIC_NOT_FOUND: "List topics with `libscope topics list`.", + CONNECTION_NOT_FOUND: "List saved connections with `libscope connections`.", + SAVED_SEARCH_NOT_FOUND: "List saved searches with `libscope searches list`.", + WEBHOOK_NOT_FOUND: "List webhooks with `libscope webhooks list`.", + LINK_NOT_FOUND: "Link IDs are shown by `libscope docs links <documentId>`.", + PACK_NOT_FOUND: "List installed packs with `libscope pack list`.", + VERSION_NOT_FOUND: "List saved versions with `libscope docs history <documentId>`.", +}; + +/** Next step to suggest for an error, if any. */ +export function errorHint(err: unknown): string | undefined { + if (err instanceof NotFoundError) return NOT_FOUND_HINTS[err.code] ?? "Check the name or ID."; + if (err instanceof ConfigError) return "Run `libscope doctor` to check your setup."; + if (err instanceof ValidationError) return "Add --help to the command to see its arguments."; + return undefined; +} + +/** Lines to print for an error. */ +export function formatError(err: unknown, verbose: boolean): string[] { + const message = err instanceof Error ? err.message : String(err); + const lines = [`✗ ${message}`]; + const hint = errorHint(err); + if (hint) lines.push(` ${hint}`); + if (verbose && err instanceof Error && err.stack) { + lines.push("", err.stack); + if (err.cause instanceof Error && err.cause.stack) lines.push("Caused by:", err.cause.stack); + } + return lines; +} + +/** Print an error to stderr and set exit code 1. */ +export function reportError(err: unknown, verbose: boolean): void { + process.stderr.write(`${formatError(err, verbose).join("\n")}\n`); + process.exitCode = 1; +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 36ceacde..41420f4b 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,452 +1,50 @@ #!/usr/bin/env node - -import { Command } from "commander"; -import { loadConfig, saveUserConfig } from "../config.js"; -import { getDatabase, runMigrations, createVectorTable, closeDatabase } from "../db/index.js"; -import { createEmbeddingProvider, type EmbeddingProvider } from "../providers/index.js"; -import { indexDocument, indexFile } from "../core/indexing.js"; -import { getSupportedExtensions } from "../core/parsers/index.js"; -import { searchDocuments, getRelatedChunks } from "../core/search.js"; -import { askQuestion, createLlmProvider } from "../core/rag.js"; -import { getDocumentRatings, listRatings } from "../core/ratings.js"; -import { createTopic, listTopics } from "../core/topics.js"; -import { getDocument, listDocuments, deleteDocument, updateDocument } from "../core/documents.js"; -import { createLink, getDocumentLinks, deleteLink, getPrerequisiteChain } from "../core/links.js"; -import type { LinkType } from "../core/links.js"; -import { getVersionHistory, rollbackToVersion } from "../core/versioning.js"; -import { initLogger, type LogLevel } from "../logger.js"; -import { readFileSync, readdirSync, statSync } from "node:fs"; -import { join, extname, basename } from "node:path"; -import { fetchAndConvert } from "../core/url-fetcher.js"; -import { spiderUrl } from "../core/spider.js"; -import type { SpiderOptions } from "../core/spider.js"; -import { exportKnowledgeBase, importFromBackup } from "../core/export.js"; -import { batchImport } from "../core/batch.js"; -import { findDuplicates } from "../core/dedup.js"; -import { - getStats, - getPopularDocuments, - getStaleDocuments, - getTopQueries, - getSearchTrends, - getSearchAnalytics, - getKnowledgeGaps, -} from "../core/analytics.js"; -import { startRepl } from "./repl.js"; -import { confirmAction } from "./confirm.js"; -import { createReporter, isVerbose } from "./reporter.js"; -import { - addTagsToDocument, - removeTagFromDocument, - listTags, - getDocumentTags, - suggestTags, -} from "../core/tags.js"; -import { bulkDelete, bulkRetag, bulkMove } from "../core/bulk.js"; -import type { BulkSelector } from "../core/bulk.js"; -import { - createWorkspace, - deleteWorkspace, - listWorkspaces, - getActiveWorkspace, - setActiveWorkspace, - getWorkspacePath, -} from "../core/workspace.js"; -import { - installPack, - removePack, - listInstalledPacks, - listAvailablePacks, - createPack, - createPackFromSource, -} from "../core/packs.js"; - -import { spawnSync } from "node:child_process"; -import { createRequire } from "node:module"; -import { FileWatcher, DEFAULT_WATCH_EXTENSIONS } from "../core/watcher.js"; -import { indexRepository, parseRepoUrl } from "../core/repo.js"; -import { - authenticateDeviceCode, - refreshAccessToken, - syncOneNote, - disconnectOneNote, -} from "../connectors/onenote.js"; -import { - loadConnectorConfig, - saveConnectorConfig, - saveNamedConnectorConfig, - loadNamedConnectorConfig, - hasNamedConnectorConfig, -} from "../connectors/index.js"; -import { syncNotion, disconnectNotion } from "../connectors/notion.js"; -import type { NotionConfig } from "../connectors/notion.js"; -import { syncSlack, disconnectSlack, type SlackConfig } from "../connectors/slack.js"; -import { - createSavedSearch, - listSavedSearches, - runSavedSearch, - deleteSavedSearch, -} from "../core/saved-searches.js"; -import { - createWebhook, - listWebhooks, - deleteWebhook, - getWebhook, - buildPayload, - signPayload, -} from "../core/webhooks.js"; -import type { WebhookEvent } from "../core/webhooks.js"; -import { registerRegistryCommands } from "./commands/registry.js"; -import { - parsePackSpecifier, - resolvePackFromRegistries, - verifyResolvedPackChecksum, - type ResolvedPack, -} from "../registry/resolve.js"; -import { loadRegistries } from "../registry/config.js"; -import { createInterface } from "node:readline"; - -// Graceful shutdown -const handleShutdown = (): void => { - closeDatabase(); - process.exit(0); -}; -process.on("SIGINT", handleShutdown); -process.on("SIGTERM", handleShutdown); - -const _require = createRequire(import.meta.url); -const _pkg = _require("../../package.json") as { version: string }; - -/** Parse a CLI option string as an integer, exiting with an error if the value is not a valid number. */ -function parseIntOption(value: string, name: string): number { - const n = Number.parseInt(value, 10); - if (Number.isNaN(n)) { - console.error(`Error: "${value}" is not a valid number for ${name}`); - process.exit(1); - } - return n; -} - -// --------------------------------------------------------------------------- -// Extracted helpers to reduce cognitive complexity (SonarCloud S3776) -// --------------------------------------------------------------------------- - -/** Truncate a string and add "..." suffix if it exceeds maxLen. */ -function previewText(text: string, maxLen: number): string { - return text.length > maxLen ? text.slice(0, maxLen) + "..." : text; -} - -/** Print context-before chunks for a search result. */ -function printContextBefore(contextBefore: Array<{ content: string }> | undefined): void { - if (!contextBefore || contextBefore.length === 0) return; - for (const c of contextBefore) { - console.log(` \u2191 ${previewText(c.content, 120)}`); - } - console.log(" \u2500 \u2500 \u2500"); -} - -/** Print context-after chunks for a search result. */ -function printContextAfter(contextAfter: Array<{ content: string }> | undefined): void { - if (!contextAfter || contextAfter.length === 0) return; - console.log(" \u2500 \u2500 \u2500"); - for (const c of contextAfter) { - console.log(` \u2193 ${previewText(c.content, 120)}`); - } -} - -/** Print a single search result with optional context chunks. */ -function printSearchResult(r: { - title: string; - score: number; - library?: string | null; - url?: string | null; - content: string; - contextBefore?: Array<{ content: string }> | undefined; - contextAfter?: Array<{ content: string }> | undefined; -}): void { - console.log(`\n\u2500\u2500 ${r.title} (score: ${r.score.toFixed(2)}) \u2500\u2500`); - if (r.library) console.log(` Library: ${r.library}`); - if (r.url) console.log(` Source: ${r.url}`); - printContextBefore(r.contextBefore); - console.log(` ${previewText(r.content, 200)}`); - printContextAfter(r.contextAfter); -} - -/** Print a single related-chunk result. */ -function printRelatedChunk(r: { - title: string; - score: number; - chunkId: string; - library?: string | null; - url?: string | null; - content: string; -}): void { - console.log(`\n\u2500\u2500 ${r.title} (score: ${r.score.toFixed(2)}) \u2500\u2500`); - console.log(` Chunk ID: ${r.chunkId}`); - if (r.library) console.log(` Library: ${r.library}`); - if (r.url) console.log(` Source: ${r.url}`); - console.log(` ${previewText(r.content, 200)}`); -} - -/** Build search filters from CLI options. */ -function buildSearchFilters(opts: { - topic?: string; - library?: string; - source?: string; -}): Record<string, unknown> | undefined { - const filters: Record<string, unknown> = {}; - if (opts.topic) filters.topic = opts.topic; - if (opts.library) filters.library = opts.library; - if (opts.source) filters.source = opts.source; - return Object.keys(filters).length > 0 ? filters : undefined; -} - -/** Report the result of a pack install operation. */ -function reportInstallResult( - result: { - packName: string; - documentsInstalled: number; - alreadyInstalled: boolean; - errors: number; - }, - reporter: { log(msg: string): void; success(msg: string): void }, - registryName?: string, -): void { - if (result.alreadyInstalled) { - reporter.log(`Pack "${result.packName}" is already installed.`); - return; - } - const errMsg = result.errors > 0 ? ` (${result.errors} errors)` : ""; - const from = registryName ? ` from ${registryName}` : ""; - reporter.success( - `Pack "${result.packName}" installed${from}: ${result.documentsInstalled} documents${errMsg}.`, - ); -} - -/** Install a resolved registry pack -- verify checksum, install, and report. */ -async function installResolvedPack( - db: ReturnType<typeof getDatabase>, - provider: EmbeddingProvider, - resolved: ResolvedPack, - installOpts: { - batchSize?: number | undefined; - resumeFrom?: number | undefined; - concurrency?: number | undefined; - }, - reporter: { - log(msg: string): void; - success(msg: string): void; - progress(c: number, t: number, l: string): void; - clearProgress(): void; - }, -): Promise<void> { - await verifyResolvedPackChecksum(resolved); - const result = await installPack(db, provider, resolved.dataPath, { - ...installOpts, - onProgress: (current, total, docTitle) => { - reporter.progress(current, total, docTitle); - }, - }); - reporter.clearProgress(); - reportInstallResult(result, reporter, resolved.registryName); -} - -/** - * Prompt user to choose a registry when there is a conflict. - * Returns the chosen source index or -1 on invalid input. - */ -async function promptRegistryChoice( - packName: string, - sources: Array<{ registryName: string; version: string; priority: number }>, -): Promise<number> { - console.log(`Pack "${packName}" found in multiple registries:`); - for (let i = 0; i < sources.length; i++) { - const s = sources[i]!; - console.log(` [${i + 1}] ${s.registryName} (v${s.version}, priority: ${s.priority})`); - } - - const rl = createInterface({ input: process.stdin, output: process.stdout }); - const answer = await new Promise<string>((resolve) => { - rl.question("Select registry [1]: ", (ans) => { - rl.close(); - resolve(ans.trim() || "1"); - }); - }); - - const choice = Number.parseInt(answer, 10) - 1; - if (Number.isNaN(choice) || choice < 0 || choice >= sources.length) return -1; - return choice; -} - /** - * Handle the conflict case where a pack exists in multiple registries. - * Returns true if the install was handled (or process exited). + * libscope CLI: program setup, global options, the error handler, and one register() call + * per command group (src/cli/commands/<group>.ts). Commands are thin adapters over the + * operations in src/core/operations. */ -async function handlePackConflict( - conflict: { - sources: Array<{ registryName: string; version: string; priority: number }>; - }, - packName: string, - version: string | undefined, - opts: { yes?: boolean }, - db: ReturnType<typeof getDatabase>, - provider: EmbeddingProvider, - installOpts: { - batchSize?: number | undefined; - resumeFrom?: number | undefined; - concurrency?: number | undefined; - }, - reporter: { - log(msg: string): void; - success(msg: string): void; - progress(c: number, t: number, l: string): void; - clearProgress(): void; - }, -): Promise<boolean> { - if (opts.yes) { - const names = conflict.sources.map((s) => s.registryName).join(", "); - reporter.log( - `Error: Pack "${packName}" found in multiple registries: ${names}. ` + - "Use --from-registry <name> to disambiguate.", - ); - closeDatabase(); - process.exit(1); - return true; - } - - const choice = await promptRegistryChoice(packName, conflict.sources); - if (choice < 0) { - reporter.log("Invalid selection. Cancelled."); - closeDatabase(); - process.exit(1); - return true; - } - - const chosen = conflict.sources[choice]!; - const retryResult = resolvePackFromRegistries(packName, { - version, - registryName: chosen.registryName, - conflictResolution: { strategy: "explicit", registryName: chosen.registryName }, - }); - - if (retryResult.resolved) { - await installResolvedPack(db, provider, retryResult.resolved, installOpts, reporter); - return true; - } - - return false; -} - -/** Try to resolve a pack from git registries; returns true if installed. */ -async function tryGitRegistryInstall( - nameOrPath: string, - opts: { fromRegistry?: string; version?: string; yes?: boolean }, - db: ReturnType<typeof getDatabase>, - provider: EmbeddingProvider, - installOpts: { - batchSize: number | undefined; - resumeFrom: number | undefined; - concurrency: number | undefined; - }, - reporter: ReturnType<typeof createReporter>, -): Promise<boolean> { - const isLocalFile = nameOrPath.endsWith(".json") || nameOrPath.endsWith(".json.gz"); - if (isLocalFile || loadRegistries().length === 0) return false; - - const { name: packName, version: specVersion } = parsePackSpecifier(nameOrPath); - const version = opts.version ?? specVersion; - - const { resolved, conflict, warnings } = resolvePackFromRegistries(packName, { - version, - registryName: opts.fromRegistry, - conflictResolution: opts.fromRegistry - ? { strategy: "explicit", registryName: opts.fromRegistry } - : opts.yes - ? { strategy: "priority" } - : undefined, - }); - - for (const w of warnings) { - reporter.log(`Warning: ${w}`); - } - - if (conflict && !resolved) { - const handled = await handlePackConflict( - conflict, - packName, - version, - opts, - db, - provider, - installOpts, - reporter, - ); - if (handled) return true; - } - - if (resolved) { - await installResolvedPack(db, provider, resolved, installOpts, reporter); - return true; - } - - return false; -} - -/** Execute a pack install from git registry, URL, or local file. */ -async function executePackInstall( - nameOrPath: string, - opts: { - registry?: string; - fromRegistry?: string; - version?: string; - yes?: boolean; - batchSize?: string; - resumeFrom?: string; - concurrency?: string; - }, -): Promise<void> { - const { db, provider } = initializeAppWithEmbedding(); - const globalOpts = program.opts<ProgramOpts>(); - const reporter = createReporter(globalOpts.verbose); - - const batchSize = opts.batchSize ? parseIntOption(opts.batchSize, "--batch-size") : undefined; - const resumeFrom = opts.resumeFrom ? parseIntOption(opts.resumeFrom, "--resume-from") : undefined; - const concurrency = opts.concurrency - ? parseIntOption(opts.concurrency, "--concurrency") - : undefined; - - if (concurrency !== undefined && concurrency < 1) { - reporter.log('Error: "--concurrency" must be an integer greater than or equal to 1.'); - closeDatabase(); - process.exit(1); - return; - } - - const installOpts = { batchSize, resumeFrom, concurrency }; - - try { - const installed = await tryGitRegistryInstall( - nameOrPath, - opts, - db, - provider, - installOpts, - reporter, - ); - if (installed) return; - - const result = await installPack(db, provider, nameOrPath, { - registryUrl: opts.registry, - ...installOpts, - onProgress: (current, total, docTitle) => { - reporter.progress(current, total, docTitle); - }, - }); - - reporter.clearProgress(); - reportInstallResult(result, reporter); - } finally { - closeDatabase(); +import { Command, Option } from "commander"; +import { realpathSync } from "node:fs"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { LOG_LEVELS } from "../config-schema.js"; +import { initLogger } from "../logger.js"; +import { + abortContext, + closeContext, + getGlobalOptions, + isAborted, + isInterruptLeftToCommand, + isWaitingForInterrupt, + setGlobalOptions, + type GlobalOptions, +} from "./context.js"; +import { reportError } from "./errors.js"; +import { register as registerAdd } from "./commands/add.js"; +import { register as registerAdmin } from "./commands/admin.js"; +import { register as registerBulk } from "./commands/bulk.js"; +import { register as registerConfig } from "./commands/config.js"; +import { register as registerConnectors } from "./commands/connectors.js"; +import { register as registerDoctor } from "./commands/doctor.js"; +import { register as registerDocs } from "./commands/docs.js"; +import { register as registerPack } from "./commands/pack.js"; +import { register as registerRegistry } from "./commands/registry.js"; +import { register as registerSearch } from "./commands/search.js"; +import { register as registerSearches } from "./commands/searches.js"; +import { register as registerServe } from "./commands/serve.js"; +import { register as registerTopics } from "./commands/topics.js"; +import { register as registerWebhooks } from "./commands/webhooks.js"; +import { register as registerWorkspace } from "./commands/workspace.js"; + +const pkg = createRequire(import.meta.url)("../../package.json") as { version: string }; + +function setupLogging(options: GlobalOptions): void { + if (options.verbose === true || process.env["LIBSCOPE_VERBOSE"] === "1") { + initLogger("debug"); + } else { + // The CLI prints its own output; structured logs only with --verbose or --log-level. + initLogger(options.logLevel ?? "silent"); } } @@ -455,2680 +53,78 @@ const program = new Command(); program .name("libscope") .description("AI-powered knowledge base with MCP integration") - .version(_pkg.version) - .option("-v, --verbose", "Enable verbose logging") - .option("--log-level <level>", "Set log level (debug, info, warn, error, silent)") - .option("--workspace <name>", "Use a specific workspace"); - -// init -program - .command("init") - .description("Initialize the LibScope database") - .action(() => { - const { config, db } = initializeApp(); - try { - // Only create vector table if provider can be initialized without download - try { - const provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); - } catch { - console.log(" ℹ Vector table skipped (embedding provider not available)"); - } - console.log(`✓ Database initialized at ${config.database.path}`); - } finally { - closeDatabase(); - } - }); - -function resolveSourceType(library?: string, topic?: string): "library" | "topic" | "manual" { - if (library) return "library"; - if (topic) return "topic"; - return "manual"; -} - -function buildSpiderOptions( - opts: { - maxPages?: number; - maxDepth?: number; - sameDomain?: boolean; - pathPrefix?: string; - excludePatterns?: string[]; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): SpiderOptions { - const options: SpiderOptions = { fetchOptions }; - if (opts.maxPages !== undefined) options.maxPages = opts.maxPages; - if (opts.maxDepth !== undefined) options.maxDepth = opts.maxDepth; - if (opts.sameDomain !== undefined) options.sameDomain = opts.sameDomain; - if (opts.pathPrefix !== undefined) options.pathPrefix = opts.pathPrefix; - if (opts.excludePatterns !== undefined) options.excludePatterns = opts.excludePatterns; - return options; -} - -async function runSpiderAdd( - db: ReturnType<typeof getDatabase>, - provider: EmbeddingProvider, - url: string, - opts: { - topic?: string; - library?: string; - version?: string; - dedup?: string; - maxPages?: number; - maxDepth?: number; - sameDomain?: boolean; - pathPrefix?: string; - excludePatterns?: string[]; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): Promise<void> { - const spiderOptions = buildSpiderOptions(opts, fetchOptions); - const sourceType = resolveSourceType(opts.library, opts.topic); - console.log(`Spidering ${url}...`); - let indexed = 0; - let failed = 0; - - const gen = spiderUrl(url, spiderOptions); - let next = await gen.next(); - while (!next.done) { - const page = next.value; - try { - await indexDocument(db, provider, { - title: page.title, - content: page.content, - sourceType, - library: opts.library, - version: opts.version, - topicId: opts.topic, - url: page.url, - dedup: opts.dedup as "skip" | "warn" | "force" | undefined, - }); - indexed++; - console.log(` ✓ [${indexed}] ${page.title} (${page.url})`); - } catch (err) { - failed++; - const msg = err instanceof Error ? err.message : String(err); - console.error(` ✗ ${page.url}: ${msg}`); - } - next = await gen.next(); - } - - const stats = next.value; - console.log( - `\nDone. Indexed: ${indexed}, Failed: ${failed}, Skipped: ${stats.pagesSkipped}` + - (stats.abortReason ? ` (stopped: ${stats.abortReason})` : ""), - ); -} - -// add -program - .command("add <fileOrUrl>") - .description( - "Index a document from a file or URL (supports: " + getSupportedExtensions().join(", ") + ")", - ) - .option("--topic <topicId>", "Assign to a topic") - .option("--library <name>", "Mark as library documentation") - .option("--version <version>", "Library version") - .option("--title <title>", "Override document title") - .option("--format <ext>", "Force file format (e.g. .pdf, .csv, .yaml)") - .option("--dedup <mode>", "Dedup mode: skip, warn, or force") - .option("--spider", "Crawl linked pages from the URL") - .option("--max-pages <n>", "Max pages to crawl (default: 25, hard cap: 200)", Number.parseInt) - .option( - "--max-depth <n>", - "Max link-hop depth from seed URL (default: 2, hard cap: 5)", - Number.parseInt, - ) - .option("--no-same-domain", "Follow links outside the seed domain") - .option("--path-prefix <path>", "Only follow links under this path prefix") - .option("--exclude-patterns <globs...>", "Glob patterns for URLs to skip") - .action( - async ( - fileOrUrl: string, - opts: { - topic?: string; - library?: string; - version?: string; - title?: string; - format?: string; - dedup?: string; - spider?: boolean; - maxPages?: number; - maxDepth?: number; - sameDomain?: boolean; - pathPrefix?: string; - excludePatterns?: string[]; - }, - ) => { - const { config, db, provider } = initializeAppWithEmbedding(); - const fetchOptions = { - allowPrivateUrls: config.indexing.allowPrivateUrls, - allowSelfSignedCerts: config.indexing.allowSelfSignedCerts, - }; - try { - if (fileOrUrl.startsWith("http://") || fileOrUrl.startsWith("https://")) { - if (opts.spider) { - await runSpiderAdd(db, provider, fileOrUrl, opts, fetchOptions); - } else { - console.log(`Fetching ${fileOrUrl}...`); - const fetched = await fetchAndConvert(fileOrUrl, fetchOptions); - const title = opts.title ?? fetched.title; - const result = await indexDocument(db, provider, { - title, - content: fetched.content, - sourceType: resolveSourceType(opts.library, opts.topic), - library: opts.library, - version: opts.version, - topicId: opts.topic, - url: fileOrUrl, - dedup: opts.dedup as "skip" | "warn" | "force" | undefined, - }); - console.log(`✓ Indexed "${title}" (${result.chunkCount} chunks)`); - console.log(` ID: ${result.id}`); - } - } else { - const result = await indexFile(db, provider, fileOrUrl, { - title: opts.title, - topic: opts.topic, - library: opts.library, - version: opts.version, - format: opts.format, - dedup: opts.dedup as "skip" | "warn" | "force" | undefined, - }); - const title = opts.title ?? basename(fileOrUrl).replace(/\.[^.]+$/, ""); - console.log(`✓ Indexed "${title}" (${result.chunkCount} chunks)`); - console.log(` ID: ${result.id}`); - } - } finally { - closeDatabase(); - } - }, - ); - -// import -program - .command("import <directory>") - .description("Bulk import files from a directory (supports all document formats)") - .option("--topic <topicId>", "Assign all to a topic") - .option("--library <name>", "Mark all as library documentation") - .option("--version <version>", "Library version") - .option( - "--extensions <exts>", - "Comma-separated file extensions to include", - ".md,.mdx,.txt,.pdf,.docx,.csv,.yaml,.yml,.json", - ) - .option("--dedup <mode>", "Dedup mode: skip, warn, or force") - .action( - async ( - directory: string, - opts: { - topic?: string; - library?: string; - version?: string; - extensions: string; - dedup?: string; - }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const extensions = new Set(opts.extensions.split(",").map((e) => e.trim().toLowerCase())); - const files = findFiles(directory, extensions); - - if (files.length === 0) { - console.log(`No matching files found in ${directory}`); - return; - } - - console.log(`Found ${files.length} files to import...`); - const startTime = Date.now(); - let indexed = 0; - let failed = 0; - - for (const file of files) { - try { - const result = await indexFile(db, provider, file, { - topic: opts.topic, - library: opts.library, - version: opts.version, - dedup: opts.dedup as "skip" | "warn" | "force" | undefined, - }); - - indexed++; - console.log( - ` [${indexed + failed}/${files.length}] ✓ ${file} (${result.chunkCount} chunks)`, - ); - } catch (err) { - failed++; - console.error( - ` [${indexed + failed}/${files.length}] ✗ ${file}: ${err instanceof Error ? err.message : String(err)}`, - ); - } - } - - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log(`\nDone: ${indexed} indexed, ${failed} failed in ${elapsed}s`); - } finally { - closeDatabase(); - } - }, - ); - -// import-batch -program - .command("import-batch <directory>") - .description("Batch import files with parallel processing") - .option("--concurrency <n>", "Number of parallel imports", "5") - .option("--filter <glob>", "Glob pattern for file selection", "**/*.{md,mdx,txt}") - .option("--dry-run", "Preview files without importing") - .option("--topic <topicId>", "Assign all to a topic") - .option("--library <name>", "Mark all as library documentation") - .option("--version <version>", "Library version") - .action( - async ( - directory: string, - opts: { - concurrency: string; - filter: string; - dryRun?: boolean; - topic?: string; - library?: string; - version?: string; - }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const { globSync } = await import("node:fs"); - const files = globSync(opts.filter, { cwd: directory }).map((f: string) => - join(directory, f), - ); - files.sort(); - - if (files.length === 0) { - console.log(`No files matching "${opts.filter}" found in ${directory}`); - return; - } - - if (opts.dryRun) { - console.log(`Found ${files.length} files (dry run):\n`); - for (const f of files) { - console.log(` ${f}`); - } - return; - } - - console.log(`Importing ${files.length} files (concurrency: ${opts.concurrency})...`); - const startTime = Date.now(); - - const result = await batchImport(db, provider, files, { - concurrency: parseIntOption(opts.concurrency, "--concurrency"), - library: opts.library, - version: opts.version, - topicId: opts.topic, - onProgress: (progress) => { - const done = progress.completed + progress.failed; - const file = progress.currentFile ?? ""; - console.log(` [${done}/${progress.total}] ${file}`); - }, - }); - - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log(`\nDone: ${result.completed} indexed, ${result.failed} failed in ${elapsed}s`); - - if (result.failed > 0) { - console.log("\nFailed files:"); - for (const r of result.results) { - if (!r.success) { - console.log(` ✗ ${r.file}: ${r.error}`); - } - } - } - } finally { - closeDatabase(); - } - }, - ); - -// search -program - .command("search <query>") - .description("Search indexed documents") - .option("--topic <topicId>", "Filter by topic") - .option("--library <name>", "Filter by library") - .option( - "--source <type>", - "Filter by source type (e.g., 'library', 'topic', 'manual', 'model-generated')", - ) - .option("--limit <n>", "Max results", "5") - .option("--offset <n>", "Offset for pagination", "0") - .option("--max-chunks-per-doc <n>", "Max chunks per document in results (default: no limit)") - .option("--context <n>", "Include N neighboring chunks before/after each result (0-2)", "0") - .option("--save <name>", "Save this search with the given name for later re-use") - .action( - async ( - query: string, - opts: { - topic?: string; - library?: string; - source?: string; - limit: string; - offset: string; - maxChunksPerDoc?: string; - context: string; - save?: string; - }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const contextChunks = parseIntOption(opts.context, "--context"); - const maxChunksPerDoc = opts.maxChunksPerDoc - ? parseIntOption(opts.maxChunksPerDoc, "--max-chunks-per-doc") - : undefined; - const { results, totalCount } = await searchDocuments(db, provider, { - query, - topic: opts.topic, - library: opts.library, - source: opts.source, - limit: parseIntOption(opts.limit, "--limit"), - offset: parseIntOption(opts.offset, "--offset"), - maxChunksPerDocument: maxChunksPerDoc, - contextChunks: contextChunks > 0 ? contextChunks : undefined, - }); - - if (results.length === 0) { - console.log("No results found."); - } else { - console.log(`\nShowing ${results.length} of ${totalCount} results:\n`); - for (const r of results) { - printSearchResult(r); - } - } - - if (opts.save) { - const filters = buildSearchFilters(opts); - const saved = createSavedSearch(db, opts.save, query, filters); - console.log(`\n✓ Search saved as "${saved.name}" (${saved.id})`); - } - } finally { - closeDatabase(); - } - }, - ); - -// related -program - .command("related <chunkId>") - .description("Find chunks related to a given chunk by vector similarity") - .option("--limit <n>", "Number of results", "10") - .option("--topic <topic>", "Filter by topic") - .option("--library <lib>", "Filter by library") - .option("--min-score <n>", "Minimum similarity score (0-1)") - .option("--tags <tags>", "Comma-separated tags to filter by") - .action( - ( - chunkId: string, - opts: { - limit: string; - topic?: string; - library?: string; - minScore?: string; - tags?: string; - }, - ) => { - const { db } = initializeApp(); - try { - const limit = parseIntOption(opts.limit, "--limit"); - const minScore = opts.minScore !== undefined ? Number.parseFloat(opts.minScore) : undefined; - const tags = opts.tags ? opts.tags.split(",").map((t) => t.trim()) : undefined; - - let result; - try { - result = getRelatedChunks(db, { - chunkId, - ...(limit !== undefined && { limit }), - ...(opts.topic !== undefined && { topic: opts.topic }), - ...(opts.library !== undefined && { library: opts.library }), - ...(tags !== undefined && { tags }), - ...(minScore !== undefined && { minScore }), - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - console.error(`Error: ${message}`); - process.exit(1); - } - - const { sourceChunk, chunks } = result; - console.log( - `\nRelated to chunk: ${sourceChunk.id} from document ${sourceChunk.documentId}`, - ); - - if (chunks.length === 0) { - console.log("No related chunks found."); - } else { - console.log(`\nShowing ${chunks.length} related chunks:\n`); - for (const r of chunks) { - printRelatedChunk(r); - } - } - } finally { - closeDatabase(); - } - }, - ); - -// saved searches -const searchesCmd = program.command("searches").description("Manage saved searches"); - -searchesCmd - .command("list") - .description("List all saved searches") - .action(() => { - const { db } = initializeApp(); - try { - const searches = listSavedSearches(db); - if (searches.length === 0) { - console.log("No saved searches."); - } else { - console.log(`Found ${searches.length} saved searches:\n`); - for (const s of searches) { - console.log(` ${s.name}`); - console.log(` ID: ${s.id}`); - console.log(` Query: "${s.query}"`); - if (s.filters) console.log(` Filters: ${JSON.stringify(s.filters)}`); - if (s.lastRunAt) console.log(` Last run: ${s.lastRunAt} (${s.resultCount} results)`); - console.log(); - } - } - } finally { - closeDatabase(); - } - }); - -searchesCmd - .command("run <nameOrId>") - .description("Run a saved search") - .action(async (nameOrId: string) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const { search, results } = await runSavedSearch(db, provider, nameOrId); - console.log(`\nRunning saved search "${search.name}" (query: "${search.query}"):\n`); - if (results.length === 0) { - console.log("No results found."); - } else { - console.log(`Found ${results.length} results:\n`); - for (const r of results) { - console.log(`── ${r.title} (score: ${r.score.toFixed(2)}) ──`); - if (r.library) console.log(` Library: ${r.library}`); - console.log(` ${r.content.slice(0, 200)}${r.content.length > 200 ? "..." : ""}`); - } - } - } finally { - closeDatabase(); - } - }); - -searchesCmd - .command("delete <nameOrId>") - .description("Delete a saved search") - .action((nameOrId: string) => { - const { db } = initializeApp(); - try { - deleteSavedSearch(db, nameOrId); - console.log(`✓ Saved search "${nameOrId}" deleted.`); - } finally { - closeDatabase(); - } - }); - -// ask (RAG question answering) -program - .command("ask <question>") - .description("Ask a question and get an LLM-synthesized answer using RAG") - .option("--top-k <n>", "Number of chunks to retrieve", "5") - .option("--topic <topic>", "Filter by topic") - .option("--library <lib>", "Filter by library") - .option("--model <model>", "Override LLM model") - .action( - async ( - question: string, - opts: { topK: string; topic?: string; library?: string; model?: string }, - ) => { - const { config, db, provider } = initializeAppWithEmbedding(); - try { - let llmProvider; - try { - if (opts.model) { - config.llm = { ...config.llm, model: opts.model }; - } - llmProvider = createLlmProvider(config); - } catch (err) { - console.error( - `\u2717 ${err instanceof Error ? err.message : String(err)}\n\n` + - `To configure an LLM provider, run:\n` + - ` libscope config set llm.provider openai # or ollama\n` + - ` export LIBSCOPE_LLM_PROVIDER=openai\n`, - ); - process.exit(1); - } - - const result = await askQuestion(db, provider, llmProvider, { - question, - topK: parseIntOption(opts.topK, "--top-k"), - topic: opts.topic, - library: opts.library, - }); - - console.log(`\n${result.answer}\n`); - - if (result.sources.length > 0) { - console.log("\u2500\u2500 Sources \u2500\u2500"); - for (const src of result.sources) { - console.log( - ` \u2022 ${src.title} (score: ${src.score.toFixed(2)}) [${src.documentId}]`, - ); - } - } - - if (result.tokensUsed != null) { - console.log(`\n Model: ${result.model} | Tokens: ${result.tokensUsed}`); - } - } finally { - closeDatabase(); - } - }, - ); - -// topics -const topicsCmd = program.command("topics").description("Manage topics"); - -topicsCmd - .command("list") - .description("List all topics") - .action(() => { - const { db } = initializeApp(); - try { - const topics = listTopics(db); - if (topics.length === 0) { - console.log("No topics found. Create one with: libscope topics create <name>"); - } else { - for (const t of topics) { - const topicDesc = t.description ? ` (${t.description})` : ""; - console.log(` ${t.id} — ${t.name}${topicDesc}`); - } - } - } finally { - closeDatabase(); - } - }); - -topicsCmd - .command("create <name>") - .description("Create a new topic") - .option("--description <desc>", "Topic description") - .option("--parent <parentId>", "Parent topic ID") - .action((name: string, opts: { description?: string; parent?: string }) => { - const { db } = initializeApp(); - try { - const topic = createTopic(db, { - name, - description: opts.description, - parentId: opts.parent, - }); - console.log(`✓ Topic created: ${topic.id} (${topic.name})`); - } finally { - closeDatabase(); - } - }); - -// ratings -const ratingsCmd = program.command("ratings").description("View document ratings"); - -ratingsCmd - .command("show <documentId>") - .description("Show ratings for a document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const doc = getDocument(db, documentId); - const summary = getDocumentRatings(db, documentId); - const ratings = listRatings(db, documentId); - - console.log(`\nRatings for: ${doc.title}`); - console.log( - ` Average: ${summary.averageRating.toFixed(1)}/5 (${summary.totalRatings} ratings)`, - ); - console.log(` Corrections suggested: ${summary.corrections}`); - - if (ratings.length > 0) { - console.log("\nRecent ratings:"); - for (const r of ratings.slice(0, 10)) { - console.log(` ${r.rating}/5 by ${r.ratedBy} — ${r.feedback ?? "(no feedback)"}`); - } - } - } finally { - closeDatabase(); - } - }); - -// links (document cross-references) -program - .command("link <sourceId> <targetId>") - .description("Create a cross-reference link between two documents") - .option("--type <type>", "Link type: see_also, prerequisite, supersedes, related", "related") - .option("--label <text>", "Human-readable description of the link") - .action((sourceId: string, targetId: string, opts: { type: string; label?: string }) => { - const validTypes = ["see_also", "prerequisite", "supersedes", "related"]; - if (!validTypes.includes(opts.type)) { - console.error(`Invalid link type: ${opts.type}. Must be one of: ${validTypes.join(", ")}`); - process.exit(1); - } - const { db } = initializeApp(); - try { - const link = createLink(db, sourceId, targetId, opts.type as LinkType, opts.label); - console.log(`✓ Link created: ${link.linkType} (${link.id})`); - } finally { - closeDatabase(); - } - }); - -program - .command("links <documentId>") - .description("Show all cross-reference links for a document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const { outgoing, incoming } = getDocumentLinks(db, documentId); - if (outgoing.length === 0 && incoming.length === 0) { - console.log("No links found for this document."); - return; - } - if (outgoing.length > 0) { - console.log("\nOutgoing links:"); - for (const l of outgoing) { - const outLinkLabel = l.label ? ` — ${l.label}` : ""; - console.log(` → [${l.linkType}] ${l.targetTitle}${outLinkLabel}`); - console.log(` ID: ${l.id} Target: ${l.targetId}`); - } - } - if (incoming.length > 0) { - console.log("\nIncoming links:"); - for (const l of incoming) { - const inLinkLabel = l.label ? ` — ${l.label}` : ""; - console.log(` ← [${l.linkType}] ${l.sourceTitle}${inLinkLabel}`); - console.log(` ID: ${l.id} Source: ${l.sourceId}`); - } - } - } finally { - closeDatabase(); - } - }); - -program - .command("unlink <linkId>") - .description("Remove a cross-reference link") - .action((linkId: string) => { - const { db } = initializeApp(); - try { - deleteLink(db, linkId); - console.log(`✓ Link ${linkId} deleted.`); - } finally { - closeDatabase(); - } - }); - -program - .command("prereqs <documentId>") - .description("Show prerequisite reading chain for a document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const chain = getPrerequisiteChain(db, documentId); - if (chain.length === 0) { - console.log("No prerequisites found for this document."); - return; - } - console.log("\nPrerequisite reading order:"); - for (let i = 0; i < chain.length; i++) { - const doc = chain[i]!; - console.log(` ${i + 1}. ${doc.title} (${doc.id})`); - } - console.log(` ${chain.length + 1}. [this document]`); - } finally { - closeDatabase(); - } - }); - -// docs -const docsCmd = program.command("docs").description("Manage documents"); - -docsCmd - .command("list") - .description("List indexed documents") - .option("--library <name>", "Filter by library") - .option("--topic <topicId>", "Filter by topic") - .option("--source-type <type>", "Filter by source type") - .option("--limit <n>", "Max results", "50") - .action((opts: { library?: string; topic?: string; sourceType?: string; limit: string }) => { - const { db } = initializeApp(); - try { - const docs = listDocuments(db, { - library: opts.library, - topicId: opts.topic, - sourceType: opts.sourceType, - limit: parseIntOption(opts.limit, "--limit"), - }); - - if (docs.length === 0) { - console.log("No documents found."); - } else { - console.log(`Found ${docs.length} documents:\n`); - for (const d of docs) { - console.log(` ${d.id} ${d.title}`); - if (d.library) { - const listDocVersion = d.version ? ` v${d.version}` : ""; - console.log(` Library: ${d.library}${listDocVersion}`); - } - if (d.url) console.log(` URL: ${d.url}`); - console.log(` Type: ${d.sourceType} | Updated: ${d.updatedAt}`); - console.log(); - } - } - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("show <documentId>") - .description("Show a specific document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const doc = getDocument(db, documentId); - console.log(`\n# ${doc.title}\n`); - console.log(`ID: ${doc.id}`); - console.log(`Type: ${doc.sourceType}`); - if (doc.library) { - const showDocVersion = doc.version ? ` v${doc.version}` : ""; - console.log(`Library: ${doc.library}${showDocVersion}`); - } - if (doc.url) console.log(`URL: ${doc.url}`); - console.log(`Submitted by: ${doc.submittedBy}`); - console.log(`Created: ${doc.createdAt}`); - console.log(`Updated: ${doc.updatedAt}`); - const tags = getDocumentTags(db, documentId); - if (tags.length > 0) { - console.log(`Tags: ${tags.map((t) => t.name).join(", ")}`); - } - console.log(`\n---\n`); - console.log(doc.content); - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("delete <documentId>") - .description("Delete a document by ID") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (documentId: string, opts: { yes?: boolean }) => { - const { db } = initializeApp(); - try { - const doc = getDocument(db, documentId); - if ( - !(await confirmAction( - `Delete document "${doc.title}" (${documentId})? This cannot be undone.`, - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - deleteDocument(db, documentId); - console.log(`✓ Deleted "${doc.title}" (${documentId})`); - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("update <documentId>") - .description("Update an existing document") - .option("--title <title>", "New title") - .option("--content <content>", "New content (will re-chunk and re-index)") - .option("--library <name>", "New library name") - .option("--version <ver>", "New version") - .option("--url <url>", "New URL") - .option("--topic <topicId>", "New topic ID") - .action( - async ( - documentId: string, - opts: { - title?: string; - content?: string; - library?: string; - version?: string; - url?: string; - topic?: string; - }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const metadata: Record<string, string | null | undefined> = {}; - if (opts.library !== undefined) metadata.library = opts.library; - if (opts.version !== undefined) metadata.version = opts.version; - if (opts.url !== undefined) metadata.url = opts.url; - if (opts.topic !== undefined) metadata.topicId = opts.topic; - - const doc = await updateDocument(db, provider, documentId, { - title: opts.title, - content: opts.content, - metadata: - Object.keys(metadata).length > 0 - ? (metadata as { - library?: string | null; - version?: string | null; - url?: string | null; - topicId?: string | null; - }) - : undefined, - }); - console.log(`✓ Updated "${doc.title}" (${doc.id})`); - } finally { - closeDatabase(); - } - }, - ); - -// tag -const tagCmd = program.command("tag").description("Manage document tags"); - -tagCmd - .command("add <docId> <tags...>") - .description("Add tags to a document (comma or space separated)") - .action((docId: string, tags: string[]) => { - const { db } = initializeApp(); - try { - const tagNames = tags - .flatMap((t) => t.split(",")) - .map((t) => t.trim()) - .filter(Boolean); - const created = addTagsToDocument(db, docId, tagNames); - console.log(`✓ Added ${created.length} tag(s) to document ${docId}`); - for (const t of created) { - console.log(` • ${t.name}`); - } - } finally { - closeDatabase(); - } - }); - -tagCmd - .command("remove <docId> <tag>") - .description("Remove a tag from a document") - .action((docId: string, tagName: string) => { - const { db } = initializeApp(); - try { - const docTags = getDocumentTags(db, docId); - const tag = docTags.find((t) => t.name === tagName.trim().toLowerCase()); - if (!tag) { - console.log(`Tag "${tagName}" not found on document ${docId}`); - return; - } - removeTagFromDocument(db, docId, tag.id); - console.log(`✓ Removed tag "${tag.name}" from document ${docId}`); - } finally { - closeDatabase(); - } - }); - -tagCmd - .command("list") - .description("List all tags with document counts") - .action(() => { - const { db } = initializeApp(); - try { - const allTags = listTags(db); - if (allTags.length === 0) { - console.log("No tags found. Add tags with: libscope tag add <docId> <tags...>"); - } else { - for (const t of allTags) { - console.log(` ${t.name} (${t.documentCount} doc${t.documentCount === 1 ? "" : "s"})`); - } - } - } finally { - closeDatabase(); - } - }); - -tagCmd - .command("suggest <documentId>") - .description("Suggest tags for a document based on content analysis") - .option("-l, --limit <n>", "Max number of suggestions", "5") - .action((documentId: string, opts: { limit: string }) => { - const { db } = initializeApp(); - try { - const limit = Number.parseInt(opts.limit, 10); - const suggestions = suggestTags(db, documentId, limit); - if (suggestions.length === 0) { - console.log("No tag suggestions found for this document."); - } else { - console.log(`Suggested tags for ${documentId}:`); - for (const tag of suggestions) { - console.log(` • ${tag}`); - } - } - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("history <documentId>") - .description("Show version history of a document") - .action((documentId: string) => { - const { db } = initializeApp(); - try { - const versions = getVersionHistory(db, documentId); - if (versions.length === 0) { - console.log("No version history found."); - } else { - console.log(`Version history for ${documentId}:\n`); - for (const v of versions) { - console.log(` v${v.version} ${v.title} (${v.createdAt})`); - } - } - } finally { - closeDatabase(); - } - }); - -docsCmd - .command("rollback <documentId> <version>") - .description("Rollback a document to a specific version") - .action(async (documentId: string, version: string) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const restored = await rollbackToVersion( - db, - provider, - documentId, - parseIntOption(version, "<version>"), - ); - console.log(`✓ Rolled back to version ${version}, saved as v${restored.version}`); - } finally { - closeDatabase(); - } - }); - -// export -program - .command("export <outputPath>") - .description("Export the knowledge base to a JSON file") - .action((outputPath: string) => { - const { db } = initializeApp(); - try { - const data = exportKnowledgeBase(db, outputPath); - console.log(`✓ Exported ${data.metadata.counts.documents} documents to ${outputPath}`); - } finally { - closeDatabase(); - } - }); - -// import-backup -program - .command("import-backup <backupPath>") - .description("Import knowledge base data from a backup file") - .action((backupPath: string) => { - const { db } = initializeApp(); - try { - const data = importFromBackup(db, backupPath); - console.log(`✓ Imported ${data.metadata.counts.documents} documents from ${backupPath}`); - } finally { - closeDatabase(); - } - }); - -// serve -program - .command("serve") - .description("Start the MCP server, REST API (--api), or web dashboard (--dashboard)") - .option("--api", "Start the REST API server instead of MCP") - .option("--dashboard", "Start the web dashboard UI") - .option("--port <port>", "Server port (default: 3378 for API, 3377 for dashboard)") - .option("--host <host>", "Server host", "localhost") - .action(async (opts: { api?: boolean; dashboard?: boolean; port?: string; host: string }) => { - if (opts.dashboard) { - const { db, provider } = initializeAppWithEmbedding(); - const { startWebServer } = await import("../web/server.js"); - const defaultPort = 3377; - const port = opts.port ? parseIntOption(opts.port, "--port") : defaultPort; - await startWebServer(db, provider, { port, host: opts.host }); - console.log(`LibScope dashboard running at http://${opts.host}:${port}`); - console.log("Press Ctrl+C to stop"); - } else if (opts.api) { - const { db, provider } = initializeAppWithEmbedding(); - const { startApiServer } = await import("../api/server.js"); - const defaultPort = 3378; - const port = opts.port ? parseIntOption(opts.port, "--port") : defaultPort; - const result = await startApiServer(db, provider, { port, host: opts.host }); - console.log(`LibScope API server listening on http://${opts.host}:${result.port}`); - console.log("Press Ctrl+C to stop"); - } else { - await import("../mcp/server.js"); - } - }); - -// config -const configCmd = program.command("config").description("Manage configuration"); - -configCmd - .command("set <key> <value>") - .description("Set a configuration value (e.g., embedding.provider local)") - .action((key: string, value: string) => { - if (key === "embedding.provider") { - if (value !== "local" && value !== "ollama" && value !== "openai") { - console.error("Invalid provider. Must be: local, ollama, or openai"); - process.exit(1); - } - saveUserConfig({ embedding: { provider: value } }); - console.log(`✓ Embedding provider set to: ${value}`); - } else if (key === "indexing.allowPrivateUrls") { - const bool = value === "true"; - saveUserConfig({ indexing: { ...loadConfig().indexing, allowPrivateUrls: bool } }); - console.log(`✓ indexing.allowPrivateUrls set to: ${bool}`); - } else if (key === "indexing.allowSelfSignedCerts") { - const bool = value === "true"; - saveUserConfig({ indexing: { ...loadConfig().indexing, allowSelfSignedCerts: bool } }); - console.log(`✓ indexing.allowSelfSignedCerts set to: ${bool}`); + .version(pkg.version) + .option("--json", "Print results as JSON") + .option("-v, --verbose", "Debug logging, and stack traces on errors") + .addOption(new Option("--log-level <level>", "Log level (default silent)").choices(LOG_LEVELS)) + .option("--workspace <name>", "Use this workspace instead of the active one") + .hook("preAction", () => { + setGlobalOptions(program.opts<GlobalOptions>()); + setupLogging(getGlobalOptions()); + }); + +registerAdd(program); +registerSearch(program); +registerDocs(program); +registerTopics(program); +registerSearches(program); +registerBulk(program); +registerConnectors(program); +registerPack(program); +registerRegistry(program); +registerServe(program); +registerConfig(program); +registerWorkspace(program); +registerWebhooks(program); +registerAdmin(program); +registerDoctor(program); + +/** Parse `argv` and run the command; errors are printed by the one handler (exit code 1). */ +export async function main(argv: readonly string[] = process.argv): Promise<void> { + try { + await program.parseAsync(argv); + } catch (err) { + if (isAborted()) { + process.stderr.write("Cancelled.\n"); + process.exitCode = 130; } else { - console.error(`Unknown config key: ${key}`); - process.exit(1); + reportError(err, getGlobalOptions().verbose === true); } - }); - -configCmd - .command("show") - .description("Show current configuration") - .action(() => { - const config = loadConfig(); - console.log(JSON.stringify(config, null, 2)); - }); - -interface ProgramOpts { - verbose?: boolean; - logLevel?: string; - workspace?: string; -} - -function setupLogging(opts: ProgramOpts): void { - if (isVerbose(opts.verbose)) { - initLogger("debug"); - } else if (opts.logLevel) { - initLogger(opts.logLevel as LogLevel); - } else { - // Default to silent in CLI mode — pretty reporter handles user-facing output. - // Set LIBSCOPE_VERBOSE=1 or pass --verbose to see structured JSON logs. - initLogger("silent"); - } -} - -/** Shared CLI initialization: loadConfig → setupLogging → getDatabase → runMigrations. */ -function initializeApp(): { - config: ReturnType<typeof loadConfig>; - db: ReturnType<typeof getDatabase>; -} { - const config = loadConfig(); - const opts = program.opts<ProgramOpts>(); - setupLogging(opts); - - if (opts.workspace) { - process.env["LIBSCOPE_WORKSPACE"] = opts.workspace; + } finally { + closeContext(); } - - const workspace = getActiveWorkspace(); - const dbPath = getWorkspacePath(workspace); - const db = getDatabase(dbPath); - runMigrations(db); - return { config, db }; -} - -/** Initialization with an embedding provider and vector table. */ -function initializeAppWithEmbedding(): { - config: ReturnType<typeof loadConfig>; - db: ReturnType<typeof getDatabase>; - provider: EmbeddingProvider; -} { - const { config, db } = initializeApp(); - const provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); - return { config, db, provider }; } -/** Recursively find files matching given extensions. */ -function findFiles(dir: string, extensions: Set<string>): string[] { - const results: string[] = []; - - function walk(currentDir: string): void { - const entries = readdirSync(currentDir); - for (const entry of entries) { - if (entry.startsWith(".")) continue; - const fullPath = join(currentDir, entry); - const stat = statSync(fullPath); - if (stat.isDirectory()) { - walk(fullPath); - } else if (extensions.has(extname(entry).toLowerCase())) { - results.push(fullPath); - } - } +/** Ctrl+C: stop the running operation; a second Ctrl+C (or nothing to stop) exits now. */ +function onInterrupt(): void { + if (isInterruptLeftToCommand()) return; + const waiting = isWaitingForInterrupt(); + if (isAborted() || !waiting) { + closeContext(); + process.exit(130); } - - walk(dir); - return results.sort((a, b) => a.localeCompare(b)); + abortContext(); } -// watch -program - .command("watch <directory>") - .description("Watch a directory for file changes and automatically re-index") - .option( - "--extensions <exts>", - "Comma-separated file extensions to watch", - DEFAULT_WATCH_EXTENSIONS.join(","), - ) - .option("--debounce <ms>", "Debounce interval in milliseconds", "300") - .action(async (directory: string, opts: { extensions: string; debounce: string }) => { - const { db, provider } = initializeAppWithEmbedding(); - const extensions = opts.extensions.split(",").map((e) => e.trim().toLowerCase()); - const debounceMs = parseIntOption(opts.debounce, "--debounce"); - - // Initial scan - const extensionSet = new Set(extensions); - const files = findFiles(directory, extensionSet); - console.log(`Found ${files.length} matching files in ${directory}`); - - let indexed = 0; - let skipped = 0; - for (const file of files) { - try { - const content = readFileSync(file, "utf-8"); - const title = basename(file).replace(/\.[^.]+$/, ""); - const result = await indexDocument(db, provider, { - title, - content, - sourceType: "manual", - url: file, - }); - if (result.chunkCount > 0) { - indexed++; - } else { - skipped++; - } - } catch { - skipped++; - } - } - console.log(`Initial scan: ${indexed} indexed, ${skipped} skipped (unchanged or failed)`); - console.log(`\nWatching for changes (extensions: ${extensions.join(", ")})...\n`); - - const watcher = new FileWatcher(db, provider, { - directory, - extensions, - debounceMs, - onIndex: (path: string): void => { - console.log(` ✓ Indexed: ${path}`); - }, - onRemove: (path: string): void => { - console.log(` ✗ Removed: ${path}`); - }, - onError: (err: Error): void => { - console.error(` ⚠ Error: ${err.message}`); - }, - }); - - const cleanup = (): void => { - watcher.stop(); - closeDatabase(); - process.exit(0); - }; - - process.on("SIGINT", cleanup); - process.on("SIGTERM", cleanup); - - watcher.start(); - - // Keep the process alive - await new Promise(() => {}); - }); - -// reindex -program - .command("reindex") - .description("Re-embed all chunks with the current embedding provider") - .option("--doc <documentId...>", "Only reindex specific document IDs") - .option("--since <date>", "Only reindex documents created on or after this ISO-8601 date") - .option("--before <date>", "Only reindex documents created on or before this ISO-8601 date") - .option("--batch-size <n>", "Chunks per embedding batch", "50") - .action(async (opts: { doc?: string[]; since?: string; before?: string; batchSize: string }) => { - const { reindex } = await import("../core/reindex.js"); - const { db, provider } = initializeAppWithEmbedding(); +/** True when this module is the process entry point (the `libscope` bin), not an import. */ +function isCliEntryPoint(): boolean { + const entry = process.argv[1]; + if (!entry) return false; + const self = realpathSync(fileURLToPath(import.meta.url)); + // Node also accepts an entry path without the ".js" extension. + return [entry, `${entry}.js`].some((candidate) => { try { - console.log("Re-embedding chunks..."); - const startTime = Date.now(); - - const result = await reindex(db, provider, { - documentIds: opts.doc, - since: opts.since, - before: opts.before, - batchSize: parseIntOption(opts.batchSize, "--batch-size"), - onProgress: (progress) => { - const done = progress.completed + progress.failed; - console.log(` [${done}/${progress.total}] chunks processed`); - }, - }); - - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log( - `\n✓ Reindex complete: ${result.completed} updated, ${result.failed} failed (${result.total} total) in ${elapsed}s`, - ); - - if (result.failedChunkIds.length > 0) { - console.log("\nFailed chunks:"); - for (const id of result.failedChunkIds) { - console.log(` ✗ ${id}`); - } - } - } finally { - closeDatabase(); + return realpathSync(candidate) === self; + } catch { + return false; } }); - -// repl (interactive search) -program - .command("repl") - .description("Start an interactive search REPL") - .option("--limit <n>", "Max results per search", "5") - .action(async (opts: { limit: string }) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - await startRepl({ db, provider, limit: parseIntOption(opts.limit, "--limit") }); - } finally { - closeDatabase(); - } - }); - -// dedupe — scan and report duplicates -program - .command("dedupe") - .description("Scan the knowledge base for duplicate documents") - .option("--threshold <n>", "Similarity threshold (0-1)", "0.95") - .option("--strategy <type>", "Detection strategy: exact, semantic, or both", "both") - .action(async (opts: { threshold: string; strategy: string }) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - console.log("Scanning for duplicates...\n"); - const groups = await findDuplicates(db, provider, { - threshold: Number.parseFloat(opts.threshold), - strategy: opts.strategy as "exact" | "semantic" | "both", - }); - - if (groups.length === 0) { - console.log("No duplicates found."); - return; - } - - console.log(`Found ${groups.length} duplicate group(s):\n`); - - const groupCol = 7; - const typeCol = 10; - const idCol = 38; - const titleCol = 40; - - const header = - "Group".padEnd(groupCol) + "Type".padEnd(typeCol) + "Document ID".padEnd(idCol) + "Title"; - console.log(header); - console.log("\u2500".repeat(header.length + titleCol)); - - for (let i = 0; i < groups.length; i++) { - const group = groups[i]!; - for (let j = 0; j < group.documentIds.length; j++) { - const gLabel = j === 0 ? String(i + 1) : ""; - const tLabel = j === 0 ? group.matchType : ""; - const docId = group.documentIds[j] ?? ""; - const title = group.titles[j] ?? ""; - console.log( - gLabel.padEnd(groupCol) + - tLabel.padEnd(typeCol) + - docId.padEnd(idCol) + - title.slice(0, titleCol), - ); - } - console.log(); - } - } finally { - closeDatabase(); - } - }); - -// stats -const statsCmd = program.command("stats").description("Usage analytics and content health metrics"); - -statsCmd - .command("overview", { isDefault: true }) - .description("Show overview dashboard") - .action(() => { - const { config, db } = initializeApp(); - try { - const s = getStats(db, config.database.path); - console.log("\n\u{1f4ca} Knowledge Base Overview\n"); - console.log(` Documents: ${s.totalDocuments}`); - console.log(` Chunks: ${s.totalChunks}`); - console.log(` Topics: ${s.totalTopics}`); - console.log(` Database size: ${(s.databaseSizeBytes / 1024).toFixed(1)} KB`); - console.log(` Total searches: ${s.totalSearches}`); - console.log(` Avg latency: ${s.avgLatencyMs} ms`); - - const trends = getSearchTrends(db, 7); - if (trends.length > 0) { - console.log("\n Recent search activity (last 7 days):"); - for (const t of trends) { - const bar = "\u2588".repeat(Math.min(t.count, 40)); - console.log(` ${t.date} ${bar} ${t.count}`); - } - } - console.log(); - } finally { - closeDatabase(); - } - }); - -statsCmd - .command("popular") - .description("Most-returned documents in search results") - .option("--limit <n>", "Number of results", "10") - .action((opts: { limit: string }) => { - const { db } = initializeApp(); - try { - const docs = getPopularDocuments(db, parseIntOption(opts.limit, "--limit")); - if (docs.length === 0) { - console.log("No search data yet."); - return; - } - console.log("\n\u{1f525} Most Popular Documents\n"); - console.log(" Hits Document"); - console.log(" " + "\u2500".repeat(50)); - for (const d of docs) { - console.log( - ` ${String(d.hitCount).padStart(4)} ${d.title} (${d.documentId.slice(0, 8)}\u2026)`, - ); - } - console.log(); - } finally { - closeDatabase(); - } - }); - -statsCmd - .command("stale") - .description("Documents with no search hits") - .option("--days <n>", "Look-back period in days", "90") - .action((opts: { days: string }) => { - const { db } = initializeApp(); - try { - const docs = getStaleDocuments(db, parseIntOption(opts.days, "--days")); - if (docs.length === 0) { - console.log("No stale documents found."); - return; - } - console.log(`\n\u{1f4ed} Stale Documents (no hits in ${opts.days} days)\n`); - console.log(" Updated Title"); - console.log(" " + "\u2500".repeat(50)); - for (const d of docs) { - const date = d.updatedAt.slice(0, 10); - console.log(` ${date} ${d.title}`); - } - console.log(); - } finally { - closeDatabase(); - } - }); - -statsCmd - .command("queries") - .description("Top search queries") - .option("--limit <n>", "Number of results", "10") - .action((opts: { limit: string }) => { - const { db } = initializeApp(); - try { - const queries = getTopQueries(db, parseIntOption(opts.limit, "--limit")); - if (queries.length === 0) { - console.log("No search data yet."); - return; - } - console.log("\n\u{1f50d} Top Search Queries\n"); - console.log(" Count Avg ms Query"); - console.log(" " + "\u2500".repeat(50)); - for (const q of queries) { - console.log( - ` ${String(q.count).padStart(5)} ${String(q.avgLatencyMs).padStart(6)} ${q.query}`, - ); - } - console.log(); - } finally { - closeDatabase(); - } - }); - -statsCmd - .command("search-analytics") - .description("Search analytics dashboard with knowledge gap detection") - .option("--days <n>", "Look-back period in days", "30") - .action((opts: { days: string }) => { - const { db } = initializeApp(); - try { - const days = parseIntOption(opts.days, "--days"); - const analytics = getSearchAnalytics(db, days); - const gaps = getKnowledgeGaps(db, days); - - console.log(`\n\u{1f50d} Search Analytics (last ${days} days)\n`); - console.log(` Total searches: ${analytics.totalSearches}`); - console.log(` Avg result count: ${analytics.avgResultCount}`); - - if (analytics.topQueries.length > 0) { - console.log("\n Top queries:"); - for (const q of analytics.topQueries) { - console.log(` ${String(q.count).padStart(5)}x ${q.query}`); - } - } - - if (analytics.zeroResultQueries.length > 0) { - console.log("\n Zero-result queries:"); - for (const q of analytics.zeroResultQueries) { - console.log(` ${String(q.count).padStart(5)}x ${q.query}`); - } - } - - if (gaps.length > 0) { - console.log("\n \u26a0\ufe0f Knowledge Gaps:"); - for (const g of gaps) { - console.log(` ${String(g.count).padStart(5)}x ${g.query} (last: ${g.lastSearched})`); - } - } - - if (analytics.queriesPerDay.length > 0) { - console.log("\n Queries per day:"); - for (const d of analytics.queriesPerDay) { - const bar = "\u2588".repeat(Math.min(d.count, 40)); - console.log(` ${d.date} ${bar} ${d.count}`); - } - } - console.log(); - } finally { - closeDatabase(); - } - }); - -// add-repo -program - .command("add-repo <url>") - .description("Index documentation from a GitHub or GitLab repository") - .option("--branch <name>", "Branch to index (default: main or from URL)") - .option("--path <subdir>", "Only index files under this subdirectory") - .option("--extensions <ext1,ext2>", "Comma-separated file extensions", ".md,.mdx,.txt,.rst") - .option("--token <pat>", "Personal access token for private repos") - .action( - async ( - url: string, - opts: { branch?: string; path?: string; extensions: string; token?: string }, - ) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - const parsed = parseRepoUrl(url); - console.log(`Repository: ${parsed.owner}/${parsed.repo} (${parsed.host})`); - - const extensions = opts.extensions.split(",").map((e) => { - const trimmed = e.trim(); - return trimmed.startsWith(".") ? trimmed : `.${trimmed}`; - }); - - const startTime = Date.now(); - const result = await indexRepository( - db, - provider, - { - url, - branch: opts.branch, - paths: opts.path ? [opts.path] : undefined, - extensions, - token: opts.token, - }, - (message: string) => { - console.log(` ${message}`); - }, - ); - - const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); - console.log(``); - console.log(`┌─────────────────────────────┐`); - console.log(`│ Repository Indexing Summary │`); - console.log(`├─────────────────────────────┤`); - console.log(`│ Indexed: ${String(result.indexed).padStart(17)} │`); - console.log(`│ Skipped: ${String(result.skipped).padStart(17)} │`); - console.log(`│ Errors: ${String(result.errors.length).padStart(17)} │`); - console.log(`│ Time: ${(elapsed + "s").padStart(17)} │`); - console.log(`└─────────────────────────────┘`); - - if (result.errors.length > 0) { - console.log("\nErrors:"); - for (const err of result.errors) { - console.log(` ✗ ${err}`); - } - } - } finally { - closeDatabase(); - } - }, - ); - -// workspace -const workspaceCmd = program.command("workspace").description("Manage workspaces"); - -workspaceCmd - .command("create <name>") - .description("Create a new workspace") - .action((name: string) => { - const ws = createWorkspace(name); - console.log(`✓ Workspace "${ws.name}" created at ${ws.path}`); - }); - -workspaceCmd - .command("list") - .description("List all workspaces") - .action(() => { - const workspaces = listWorkspaces(); - const active = getActiveWorkspace(); - - if (workspaces.length === 0) { - console.log("No workspaces found."); - return; - } - - console.log("Workspaces:\n"); - for (const ws of workspaces) { - const marker = ws.name === active ? " (active)" : ""; - console.log(` ${ws.name}${marker}`); - console.log(` Path: ${ws.path}`); - console.log(` Created: ${ws.createdAt}`); - console.log(); - } - }); - -workspaceCmd - .command("use <name>") - .description("Switch active workspace") - .action((name: string) => { - setActiveWorkspace(name); - console.log(`✓ Switched to workspace "${name}"`); - }); - -workspaceCmd - .command("delete <name>") - .description("Delete a workspace") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (name: string, opts: { yes?: boolean }) => { - if (!(await confirmAction(`Delete workspace "${name}"? This cannot be undone.`, !!opts.yes))) { - console.log("Cancelled."); - return; - } - deleteWorkspace(name); - console.log(`✓ Workspace "${name}" deleted.`); - }); - -// pack -const packCmd = program.command("pack").description("Manage knowledge packs"); - -packCmd - .command("install <nameOrPath>") - .description( - "Install a knowledge pack from a git registry, URL registry, or local .json/.json.gz file. " + - "Supports name@version syntax.", - ) - .option("--registry <url>", "Custom registry URL (for URL-based registries)") - .option("--from-registry <name>", "Install from a specific git registry by name") - .option("--version <semver>", "Install a specific version (for git registries)") - .option("-y, --yes", "Non-interactive mode (fail on conflicts instead of prompting)") - .option("--batch-size <n>", "Number of documents to embed per batch (default: 10)") - .option("--resume-from <n>", "Skip the first N documents (resume a partial install)") - .option("--concurrency <n>", "Number of batches to embed in parallel (default: 4)") - .action( - async ( - nameOrPath: string, - opts: { - registry?: string; - fromRegistry?: string; - version?: string; - yes?: boolean; - batchSize?: string; - resumeFrom?: string; - concurrency?: string; - }, - ) => { - await executePackInstall(nameOrPath, opts); - }, - ); - -packCmd - .command("remove <name>") - .description("Remove a pack and its documents") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (name: string, opts: { yes?: boolean }) => { - if ( - !(await confirmAction( - `Remove pack "${name}" and its documents? This cannot be undone.`, - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - const { db } = initializeApp(); - try { - removePack(db, name); - console.log(`✓ Pack "${name}" removed.`); - } finally { - closeDatabase(); - } - }); - -packCmd - .command("list") - .description("List installed or available packs") - .option("--available", "List packs available in the registry") - .option("--registry <url>", "Custom registry URL") - .action(async (opts: { available?: boolean; registry?: string }) => { - if (opts.available) { - const packs = await listAvailablePacks(opts.registry); - if (packs.length === 0) { - console.log("No packs available in the registry."); - return; - } - console.log("Available packs:\n"); - for (const p of packs) { - console.log(` ${p.name} v${p.version} — ${p.description} (${p.docCount} docs)`); - } - } else { - const { db } = initializeApp(); - try { - const packs = listInstalledPacks(db); - if (packs.length === 0) { - console.log("No packs installed."); - return; - } - console.log("Installed packs:\n"); - for (const p of packs) { - console.log(` ${p.name} v${p.version} — ${p.description ?? ""} (${p.docCount} docs)`); - } - } finally { - closeDatabase(); - } - } - }); - -packCmd - .command("create") - .description("Create a pack from the database, a local folder, or a URL") - .requiredOption("--name <name>", "Pack name") - .option("--from <sources...>", "Source folder(s), file(s), or URL(s) to build pack from") - .option("--topic <topic>", "Filter documents by topic ID (database mode only)") - .option("--version <version>", "Pack version (default: 1.0.0)") - .option("--description <desc>", "Pack description") - .option("--author <author>", "Pack author") - .option("--output <path>", "Output file path") - .option( - "--extensions <exts>", - "Comma-separated file extensions to include (e.g. .md,.html). Default: all supported", - ) - .option("--exclude <patterns...>", "Glob patterns to exclude (e.g. *.min.js, assets/**)") - .option("--no-recursive", "Do not recurse into subdirectories") - .action( - async (opts: { - name: string; - from?: string[]; - topic?: string; - version?: string; - description?: string; - author?: string; - output?: string; - extensions?: string; - exclude?: string[]; - recursive: boolean; - }) => { - if (opts.from && opts.from.length > 0) { - // Source mode: build pack directly from files/URLs (no database needed) - // Default to .json.gz for source packs (they can be large) - const outputPath = opts.output ?? `${opts.name}.json.gz`; - const extensionList = opts.extensions - ? opts.extensions.split(",").map((e) => e.trim()) - : undefined; - - const pack = await createPackFromSource({ - name: opts.name, - from: opts.from, - version: opts.version, - description: opts.description, - author: opts.author, - outputPath, - extensions: extensionList, - exclude: opts.exclude, - recursive: opts.recursive, - onProgress: ({ file, index, total }) => { - const pct = Math.round(((index + 1) / total) * 100); - const short = file.length > 60 ? `...${file.slice(-57)}` : file; - process.stdout.write(`\r [${pct}%] ${index + 1}/${total} ${short}`.padEnd(80)); - }, - }); - // Clear progress line - process.stdout.write("\r" + " ".repeat(80) + "\r"); - console.log( - `✓ Pack "${pack.name}" created with ${pack.documents.length} documents → ${outputPath}`, - ); - } else { - // Database mode: export existing documents - const outputPath = opts.output ?? `${opts.name}.json`; - const { db } = initializeApp(); - try { - const pack = createPack(db, { - name: opts.name, - version: opts.version, - description: opts.description, - author: opts.author, - topic: opts.topic, - outputPath, - }); - console.log( - `✓ Pack "${pack.name}" created with ${pack.documents.length} documents → ${outputPath}`, - ); - } finally { - closeDatabase(); - } - } - }, - ); - -/** Perform device-code auth for OneNote and persist credentials. */ -async function onenoteDeviceAuth( - onenoteConf: Record<string, unknown>, - connConfig: Record<string, unknown>, -): Promise<{ token: string; conf: Record<string, unknown> }> { - const clientId = (onenoteConf.clientId as string | undefined) ?? process.env.ONENOTE_CLIENT_ID; - if (!clientId) { - console.error("Error: No client ID. Set ONENOTE_CLIENT_ID env var or provide --token."); - process.exit(1); - } - const tenantId = - (onenoteConf.tenantId as string | undefined) ?? process.env.ONENOTE_TENANT_ID ?? "common"; - - console.log("Starting device code authentication..."); - const auth = await authenticateDeviceCode(clientId, tenantId); - const updated = { - ...onenoteConf, - clientId, - tenantId, - accessToken: auth.accessToken, - refreshToken: auth.refreshToken, - tokenExpiry: auth.expiresAt, - }; - connConfig.onenote = updated; - saveConnectorConfig(connConfig); - console.log("\u2713 Authenticated successfully"); - return { token: auth.accessToken, conf: updated }; } -/** Refresh an existing OneNote access token and persist credentials. */ -async function onenoteRefreshAuth( - onenoteConf: Record<string, unknown>, - connConfig: Record<string, unknown>, -): Promise<string | undefined> { - const clientId = onenoteConf.clientId as string | undefined; - const refreshTok = onenoteConf.refreshToken as string | undefined; - if (!clientId || !refreshTok) { - return onenoteConf.accessToken as string | undefined; - } - const tenantId = (onenoteConf.tenantId as string | undefined) ?? "common"; - const auth = await refreshAccessToken(clientId, refreshTok, tenantId); - onenoteConf.accessToken = auth.accessToken; - onenoteConf.refreshToken = auth.refreshToken; - onenoteConf.tokenExpiry = auth.expiresAt; - connConfig.onenote = onenoteConf; - saveConnectorConfig(connConfig); - return auth.accessToken; +if (isCliEntryPoint()) { + process.on("SIGINT", onInterrupt); + process.on("SIGTERM", onInterrupt); + await main(); } -// connect onenote -const connectCmd = program.command("connect").description("Connect external services"); - -connectCmd - .command("onenote") - .description("Connect and sync OneNote notebooks via Microsoft Graph API") - .option("--token <accessToken>", "Use a pre-existing access token") - .option("--sync", "Incremental re-sync only") - .option("--notebook <name>", "Sync a specific notebook") - .action(async (opts: { token?: string; sync?: boolean; notebook?: string }) => { - const config = loadConfig(); - setupLogging(program.opts<ProgramOpts>()); - - const workspace = program.opts().workspace as string | undefined; - if (workspace) { - process.env["LIBSCOPE_WORKSPACE"] = workspace; - } - const wsName = getActiveWorkspace(); - const wsPath = getWorkspacePath(wsName); - const dbPath = join(wsPath, config.database.path); - const db = getDatabase(dbPath); - runMigrations(db); - const provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); - - try { - const connConfig = loadConnectorConfig(); - let onenoteConf = (connConfig.onenote ?? {}) as Record<string, unknown>; - - let accessToken = opts.token; - - if (!accessToken && !opts.sync) { - const result = await onenoteDeviceAuth(onenoteConf, connConfig); - accessToken = result.token; - onenoteConf = result.conf; - } - - if (opts.sync && !accessToken) { - accessToken = await onenoteRefreshAuth(onenoteConf, connConfig); - } - - if (!accessToken) { - console.error("Error: No access token available. Run without --sync to authenticate."); - process.exit(1); - } - - const notebooks = opts.notebook ? [opts.notebook] : ["all"]; - const syncResult = await syncOneNote(db, provider, { - clientId: (onenoteConf.clientId as string) ?? "", - tenantId: (onenoteConf.tenantId as string) ?? "common", - accessToken, - notebooks, - excludeSections: (onenoteConf.excludeSections as string[]) ?? [], - lastSync: opts.sync ? (onenoteConf.lastSync as string | undefined) : undefined, - }); - - console.log(`\n✓ OneNote sync complete:`); - console.log(` Notebooks: ${syncResult.notebooks}`); - console.log(` Sections: ${syncResult.sections}`); - console.log(` Added: ${syncResult.pagesAdded}`); - console.log(` Updated: ${syncResult.pagesUpdated}`); - console.log(` Deleted: ${syncResult.pagesDeleted}`); - if (syncResult.errors.length > 0) { - console.log(` Errors: ${syncResult.errors.length}`); - for (const e of syncResult.errors) { - console.log(` - ${e.page}: ${e.error}`); - } - } - } finally { - db.close(); - } - }); - -// disconnect commands -const disconnectCmd = program.command("disconnect").description("Disconnect external sources"); - -disconnectCmd - .command("onenote") - .description("Disconnect OneNote and remove its data") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (opts: { yes?: boolean }) => { - if ( - !(await confirmAction( - "Disconnect OneNote and remove all its data? This cannot be undone.", - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - const config = loadConfig(); - setupLogging(program.opts<ProgramOpts>()); - - const workspace2 = program.opts().workspace as string | undefined; - if (workspace2) { - process.env["LIBSCOPE_WORKSPACE"] = workspace2; - } - const wsName2 = getActiveWorkspace(); - const wsPath = getWorkspacePath(wsName2); - const dbPath = join(wsPath, config.database.path); - const db = getDatabase(dbPath); - runMigrations(db); - - try { - const removed = disconnectOneNote(db); - console.log(`✓ Disconnected OneNote. Removed ${removed} documents.`); - } finally { - db.close(); - } - }); - -connectCmd - .command("obsidian <vault-path>") - .description("Sync an Obsidian vault into the knowledge base") - .option("--sync", "Incremental re-sync (only changed files)") - .option( - "--topic-mapping <mode>", - "Map topics from 'folder' or 'frontmatter' (default: folder)", - "folder", - ) - .option("--exclude <patterns...>", "Additional exclude patterns") - .action( - async ( - vaultPath: string, - cmdOpts: { sync?: boolean; topicMapping?: string; exclude?: string[] }, - ) => { - const { config, db } = initializeApp(); - const provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); - - try { - const { syncObsidianVault } = await import("../connectors/obsidian.js"); - - const topicMapping: "folder" | "frontmatter" = - cmdOpts.topicMapping === "frontmatter" ? "frontmatter" : "folder"; - const obsConfig = { - vaultPath: join(process.cwd(), vaultPath).replace(/\/+$/, ""), - topicMapping, - excludePatterns: cmdOpts.exclude ?? [], - }; - - // Use absolute path if provided - if (vaultPath.startsWith("/")) { - obsConfig.vaultPath = vaultPath; - } - - console.log(`Syncing Obsidian vault: ${obsConfig.vaultPath}`); - const result = await syncObsidianVault(db, provider, obsConfig); - - console.log(`✓ Sync complete:`); - console.log(` Added: ${result.added}`); - console.log(` Updated: ${result.updated}`); - console.log(` Deleted: ${result.deleted}`); - if (result.errors.length > 0) { - console.log(` Errors: ${result.errors.length}`); - for (const e of result.errors) { - console.log(` - ${e.file}: ${e.error}`); - } - } - } finally { - closeDatabase(); - } - }, - ); - -connectCmd - .command("slack") - .description("Connect a Slack workspace") - .option("--token <token>", "Slack bot or user token (xoxb-... or xoxp-...)") - .option("--channels <channels>", "Comma-separated channel names or IDs, or 'all'", "all") - .option("--exclude <channels>", "Comma-separated channel names to exclude") - .option( - "--thread-mode <mode>", - "Thread handling: aggregate (full thread as one doc) or separate", - "aggregate", - ) - .option("--sync", "Sync using saved configuration") - .action( - async (opts: { - token?: string; - channels: string; - exclude?: string; - threadMode: string; - sync?: boolean; - }) => { - const { db, provider } = initializeAppWithEmbedding(); - try { - let slackConfig: SlackConfig; - - if (opts.sync) { - if (!hasNamedConnectorConfig("slack")) { - console.error( - "No Slack configuration found. Run 'libscope connect slack --token ...' first.", - ); - process.exit(1); - } - slackConfig = loadNamedConnectorConfig<SlackConfig>("slack"); - } else { - if (!opts.token) { - console.error("--token is required for initial Slack connection."); - process.exit(1); - } - slackConfig = { - token: opts.token, - channels: opts.channels.split(",").map((c) => c.trim()), - threadMode: opts.threadMode === "separate" ? "separate" : "aggregate", - }; - if (opts.exclude) { - slackConfig = { - ...slackConfig, - excludeChannels: opts.exclude.split(",").map((c) => c.trim()), - }; - } - } - - console.log("Syncing Slack messages..."); - const result = await syncSlack(db, provider, slackConfig); - - const updatedConfig: SlackConfig = { - ...slackConfig, - lastSync: new Date().toISOString(), - }; - saveNamedConnectorConfig("slack", updatedConfig); - - console.log(`✓ Slack sync complete:`); - console.log(` Channels: ${result.channels}`); - console.log(` Messages indexed: ${result.messagesIndexed}`); - console.log(` Threads indexed: ${result.threadsIndexed}`); - if (result.errors.length > 0) { - console.log(` Errors: ${result.errors.length}`); - for (const err of result.errors) { - console.log(` - #${err.channel}: ${err.error}`); - } - } - } finally { - closeDatabase(); - } - }, - ); - -connectCmd - .command("confluence") - .description("Sync Confluence spaces and pages") - .option("--url <url>", "Confluence base URL (e.g. https://acme.atlassian.net)") - .option("--type <type>", "Confluence type: cloud or server (Data Center)", "cloud") - .option("--email <email>", "Confluence user email (Cloud only)") - .option("--token <token>", "API token (Cloud) or Personal Access Token (Server/Data Center)") - .option("--spaces <keys>", "Comma-separated space keys, or 'all'", "all") - .option("--exclude-spaces <keys>", "Comma-separated space keys to exclude") - .option("--sync", "Sync using previously saved config") - .action( - async (opts: { - url?: string; - type?: string; - email?: string; - token?: string; - spaces?: string; - excludeSpaces?: string; - sync?: boolean; - }) => { - const { syncConfluence } = await import("../connectors/confluence.js"); - const { db, provider } = initializeAppWithEmbedding(); - try { - const url = opts.url ?? process.env["CONFLUENCE_URL"] ?? ""; - const confluenceType = opts.type === "server" ? "server" : ("cloud" as "cloud" | "server"); - const email = opts.email ?? process.env["CONFLUENCE_EMAIL"] ?? undefined; - const token = opts.token ?? process.env["CONFLUENCE_TOKEN"] ?? ""; - - const spaces = (opts.spaces ?? "all").split(",").map((s) => s.trim()); - const excludeSpaces = opts.excludeSpaces - ? opts.excludeSpaces.split(",").map((s) => s.trim()) - : undefined; - - const result = await syncConfluence(db, provider, { - baseUrl: url, - type: confluenceType, - ...(email ? { email } : {}), - token, - spaces, - excludeSpaces, - }); - - console.log(`✓ Confluence sync complete`); - console.log(` Spaces: ${result.spaces}`); - console.log(` Pages indexed: ${result.pagesIndexed}`); - console.log(` Pages updated: ${result.pagesUpdated}`); - if (result.errors.length > 0) { - console.log(` Errors: ${result.errors.length}`); - for (const e of result.errors) { - console.log(` - ${e.page}: ${e.error}`); - } - } - } finally { - closeDatabase(); - } - }, - ); - -disconnectCmd - .command("obsidian <vault-path>") - .description("Remove all documents from an Obsidian vault") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (vaultPath: string, opts: { yes?: boolean }) => { - if ( - !(await confirmAction( - `Disconnect Obsidian vault "${vaultPath}" and remove its documents? This cannot be undone.`, - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - const { db } = initializeApp(); - try { - const { disconnectVault } = await import("../connectors/obsidian.js"); - - let resolvedPath = vaultPath; - if (!vaultPath.startsWith("/")) { - resolvedPath = join(process.cwd(), vaultPath); - } - - const removed = disconnectVault(db, resolvedPath); - console.log(`✓ Disconnected vault. Removed ${removed} documents.`); - } finally { - closeDatabase(); - } - }); - -connectCmd - .command("notion") - .description("Connect and sync Notion pages and databases") - .option("--token <token>", "Notion integration token (secret_...)") - .option("--sync", "Sync pages using a previously stored token") - .option("--exclude <ids...>", "Page/database IDs to exclude") - .action(async (opts: { token?: string; sync?: boolean; exclude?: string[] }) => { - const { db, provider } = initializeAppWithEmbedding(); - let token = opts.token; - - if (opts.sync && !token) { - token = process.env["NOTION_TOKEN"]; - if (!token) { - console.error("Error: --token is required, or set NOTION_TOKEN environment variable."); - process.exitCode = 1; - return; - } - } - - if (!token) { - console.error("Error: --token <token> is required."); - process.exitCode = 1; - return; - } - - const config: NotionConfig = { token }; - if (opts.exclude) { - config.excludePages = opts.exclude; - } - - console.log("Syncing Notion..."); - const result = await syncNotion(db, provider, config); - console.log( - `✓ Synced: ${result.pagesIndexed} pages, ${result.databasesIndexed} databases` + - (result.errors.length > 0 ? `, ${result.errors.length} errors` : ""), - ); - for (const err of result.errors) { - console.log(` ⚠ ${err.page}: ${err.error}`); - } - }); - -disconnectCmd - .command("notion") - .description("Remove all Notion documents") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (opts: { yes?: boolean }) => { - if ( - !(await confirmAction( - "Disconnect Notion and remove all its documents? This cannot be undone.", - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - const { db } = initializeApp(); - try { - const removed = await disconnectNotion(db); - console.log(`✓ Removed ${removed} Notion documents.`); - } finally { - closeDatabase(); - } - }); - -disconnectCmd - .command("slack") - .description("Remove all Slack data from the knowledge base") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (opts: { yes?: boolean }) => { - if ( - !(await confirmAction( - "Disconnect Slack and remove all its data? This cannot be undone.", - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - const { db } = initializeApp(); - try { - const count = disconnectSlack(db); - console.log(`✓ Removed ${count} Slack documents from the knowledge base.`); - } finally { - closeDatabase(); - } - }); - -disconnectCmd - .command("confluence") - .description("Remove all Confluence-synced content") - .option("-y, --yes", "Skip confirmation prompt") - .action(async (opts: { yes?: boolean }) => { - if ( - !(await confirmAction( - "Disconnect Confluence and remove all synced content? This cannot be undone.", - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - const { disconnectConfluence } = await import("../connectors/confluence.js"); - const { db } = initializeApp(); - try { - const removed = disconnectConfluence(db); - console.log(`✓ Removed ${removed} Confluence documents`); - } finally { - closeDatabase(); - } - }); - -// update -program - .command("update") - .description("Update libscope to the latest version") - .action(() => { - const currentVersion = _pkg.version; - console.log(`Current version: ${currentVersion}`); - - try { - const versionResult = spawnSync("npm", ["view", "libscope", "version"], { - encoding: "utf-8", - }); - if (versionResult.error || versionResult.status !== 0) { - throw new Error(versionResult.stderr?.trim() || "Failed to check latest version"); - } - const latest = versionResult.stdout.trim(); - if (latest === currentVersion) { - console.log("✓ Already up to date."); - return; - } - console.log(`Latest version: ${latest}`); - console.log("Updating..."); - const installResult = spawnSync("npm", ["install", "-g", "libscope@latest"], { - stdio: "inherit", - }); - if (installResult.error || installResult.status !== 0) { - throw new Error("npm install failed"); - } - console.log(`✓ Updated to ${latest}`); - } catch (err) { - console.error("Failed to update. Try manually: npm install -g libscope@latest"); - if (err instanceof Error) console.error(err.message); - process.exit(1); - } - }); - -// bulk -const bulkCmd = program.command("bulk").description("Bulk operations on documents"); - -bulkCmd - .command("delete") - .description("Delete multiple documents matching filters") - .option("--topic <topicId>", "Filter by topic ID") - .option("--library <name>", "Filter by library name") - .option("--source-type <type>", "Filter by source type") - .option("--tags <tags>", "Filter by tags (comma-separated)") - .option("--dry-run", "Show what would be affected without making changes") - .option("-y, --yes", "Skip confirmation prompt") - .action( - async (opts: { - topic?: string; - library?: string; - sourceType?: string; - tags?: string; - dryRun?: boolean; - yes?: boolean; - }) => { - const { db } = initializeApp(); - try { - const selector: BulkSelector = {}; - if (opts.topic) selector.topicId = opts.topic; - if (opts.library) selector.library = opts.library; - if (opts.sourceType) selector.sourceType = opts.sourceType; - if (opts.tags) selector.tags = opts.tags.split(",").map((t) => t.trim()); - - const result = bulkDelete(db, selector, true); - console.log(`Found ${result.affected} document(s) matching filters.`); - - if (result.affected === 0) return; - - if (opts.dryRun) { - for (const id of result.documentIds) { - console.log(` - ${id}`); - } - console.log("(dry run — no changes made)"); - return; - } - - if ( - !(await confirmAction( - `Delete ${result.affected} document(s)? This cannot be undone.`, - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - - const actual = bulkDelete(db, selector); - console.log(`✓ Deleted ${actual.affected} document(s).`); - } finally { - closeDatabase(); - } - }, - ); - -bulkCmd - .command("retag") - .description("Add or remove tags from multiple documents") - .option("--topic <topicId>", "Filter by topic ID") - .option("--library <name>", "Filter by library name") - .option("--source-type <type>", "Filter by source type") - .option("--tags <tags>", "Filter by tags (comma-separated)") - .option("--add-tags <tags>", "Tags to add (comma-separated)") - .option("--remove-tags <tags>", "Tags to remove (comma-separated)") - .option("--dry-run", "Show what would be affected without making changes") - .option("-y, --yes", "Skip confirmation prompt") - .action( - async (opts: { - topic?: string; - library?: string; - sourceType?: string; - tags?: string; - addTags?: string; - removeTags?: string; - dryRun?: boolean; - yes?: boolean; - }) => { - const { db } = initializeApp(); - try { - const selector: BulkSelector = {}; - if (opts.topic) selector.topicId = opts.topic; - if (opts.library) selector.library = opts.library; - if (opts.sourceType) selector.sourceType = opts.sourceType; - if (opts.tags) selector.tags = opts.tags.split(",").map((t) => t.trim()); - - const addTags = opts.addTags ? opts.addTags.split(",").map((t) => t.trim()) : undefined; - const removeTags = opts.removeTags - ? opts.removeTags.split(",").map((t) => t.trim()) - : undefined; - - const result = bulkRetag(db, selector, addTags, removeTags, true); - console.log(`Found ${result.affected} document(s) matching filters.`); - - if (result.affected === 0) return; - - if (opts.dryRun) { - for (const id of result.documentIds) { - console.log(` - ${id}`); - } - console.log("(dry run — no changes made)"); - return; - } - - if (!(await confirmAction(`Retag ${result.affected} document(s)?`, !!opts.yes))) { - console.log("Cancelled."); - return; - } - - const actual = bulkRetag(db, selector, addTags, removeTags); - console.log(`✓ Retagged ${actual.affected} document(s).`); - } finally { - closeDatabase(); - } - }, - ); - -bulkCmd - .command("move") - .description("Move multiple documents to a different topic") - .option("--topic <topicId>", "Filter by topic ID") - .option("--library <name>", "Filter by library name") - .option("--source-type <type>", "Filter by source type") - .option("--tags <tags>", "Filter by tags (comma-separated)") - .requiredOption("--to <targetTopicId>", "Target topic ID to move documents to") - .option("--dry-run", "Show what would be affected without making changes") - .option("-y, --yes", "Skip confirmation prompt") - .action( - async (opts: { - topic?: string; - library?: string; - sourceType?: string; - tags?: string; - to: string; - dryRun?: boolean; - yes?: boolean; - }) => { - const { db } = initializeApp(); - try { - const selector: BulkSelector = {}; - if (opts.topic) selector.topicId = opts.topic; - if (opts.library) selector.library = opts.library; - if (opts.sourceType) selector.sourceType = opts.sourceType; - if (opts.tags) selector.tags = opts.tags.split(",").map((t) => t.trim()); - - const result = bulkMove(db, selector, opts.to, true); - console.log(`Found ${result.affected} document(s) matching filters.`); - - if (result.affected === 0) return; - - if (opts.dryRun) { - for (const id of result.documentIds) { - console.log(` - ${id}`); - } - console.log("(dry run — no changes made)"); - return; - } - - if ( - !(await confirmAction( - `Move ${result.affected} document(s) to topic "${opts.to}"?`, - !!opts.yes, - )) - ) { - console.log("Cancelled."); - return; - } - - const actual = bulkMove(db, selector, opts.to); - console.log(`✓ Moved ${actual.affected} document(s) to topic "${opts.to}".`); - } finally { - closeDatabase(); - } - }, - ); - -// Webhooks -const webhookCmd = program.command("webhooks").description("Manage webhooks"); - -webhookCmd - .command("list") - .description("List all registered webhooks") - .action(() => { - const { db } = initializeApp(); - try { - const hooks = listWebhooks(db); - if (hooks.length === 0) { - console.log("No webhooks registered."); - } else { - console.log(`Found ${hooks.length} webhook(s):\n`); - for (const h of hooks) { - console.log(` ${h.url}`); - console.log(` ID: ${h.id}`); - console.log(` Events: ${h.events.join(", ")}`); - console.log(` Active: ${h.active ? "yes" : "no"}`); - if (h.lastTriggeredAt) console.log(` Last triggered: ${h.lastTriggeredAt}`); - if (h.failureCount > 0) console.log(` Failures: ${h.failureCount}`); - console.log(); - } - } - } finally { - closeDatabase(); - } - }); - -webhookCmd - .command("create <url>") - .description("Register a new webhook") - .requiredOption( - "--events <events>", - "Comma-separated event types (e.g. document.created,document.updated)", - ) - .option("--secret <secret>", "Secret for HMAC-SHA256 signature") - .action(async (url: string, opts: { events: string; secret?: string }) => { - const { db } = initializeApp(); - try { - const events = opts.events.split(",").map((e) => e.trim()) as WebhookEvent[]; - const webhook = await createWebhook(db, url, events, opts.secret); - console.log(`✓ Webhook created: ${webhook.id}`); - console.log(` URL: ${webhook.url}`); - console.log(` Events: ${webhook.events.join(", ")}`); - } finally { - closeDatabase(); - } - }); - -webhookCmd - .command("delete <id>") - .description("Delete a webhook") - .action((id: string) => { - const { db } = initializeApp(); - try { - deleteWebhook(db, id); - console.log(`✓ Webhook "${id}" deleted.`); - } finally { - closeDatabase(); - } - }); - -webhookCmd - .command("test <id>") - .description("Send a test ping to a webhook") - .action(async (id: string) => { - const { db } = initializeApp(); - try { - const webhook = getWebhook(db, id); - const body = buildPayload("document.created", { test: true, message: "Webhook test ping" }); - const headers: Record<string, string> = { "Content-Type": "application/json" }; - if (webhook.secret) { - headers["X-LibScope-Signature"] = signPayload(body, webhook.secret); - } - console.log(`Sending test ping to ${webhook.url}...`); - const resp = await fetch(webhook.url, { - method: "POST", - headers, - body, - signal: AbortSignal.timeout(5000), - }); - console.log(`✓ Response: ${resp.status} ${resp.statusText}`); - } finally { - closeDatabase(); - } - }); - -// schedule -const scheduleCmd = program.command("schedule").description("Manage connector sync schedules"); - -scheduleCmd - .command("list") - .description("List all configured connector sync schedules") - .action(async () => { - const { loadScheduleEntries: loadEntries } = await import("../core/scheduler.js"); - const entries = loadEntries(); - if (entries.length === 0) { - console.log("No scheduled syncs configured."); - console.log( - 'Add a schedule to a connector config: libscope schedule set <connector> "<cron>"', - ); - return; - } - console.log("Connector sync schedules:\n"); - for (const entry of entries) { - console.log(` ${entry.connectorName} (${entry.connectorType})`); - console.log(` Cron: ${entry.cronExpression}`); - console.log(); - } - }); - -scheduleCmd - .command("set <connector> <cron>") - .description('Set a sync schedule for a connector (e.g. schedule set notion "0 */6 * * *")') - .action(async (connector: string, cronExpr: string) => { - const nodeCron = await import("node-cron"); - if (!nodeCron.validate(cronExpr)) { - console.error(`Invalid cron expression: "${cronExpr}"`); - console.error("Examples: '0 */6 * * *' (every 6h), '0 0 * * *' (daily at midnight)"); - process.exit(1); - } - - const { - loadNamedConnectorConfig: loadCfg, - saveNamedConnectorConfig: saveCfg, - hasNamedConnectorConfig: hasCfg, - } = await import("../connectors/index.js"); - - if (!hasCfg(connector)) { - console.error( - `No connector config found for "${connector}". Run 'libscope connect ${connector}' first.`, - ); - process.exit(1); - } - - const config = loadCfg<Record<string, unknown>>(connector); - config.schedule = { cronExpression: cronExpr }; - saveCfg(connector, config); - console.log(`✓ Schedule set for ${connector}: ${cronExpr}`); - console.log( - "The schedule will be active when the API server is running (libscope serve --api)", - ); - }); - -scheduleCmd - .command("remove <connector>") - .description("Remove the sync schedule for a connector") - .action(async (connector: string) => { - const { - loadNamedConnectorConfig: loadCfg, - saveNamedConnectorConfig: saveCfg, - hasNamedConnectorConfig: hasCfg, - } = await import("../connectors/index.js"); - - if (!hasCfg(connector)) { - console.error(`No connector config found for "${connector}".`); - process.exit(1); - } - - const config = loadCfg<Record<string, unknown>>(connector); - if (!config.schedule) { - console.error(`No schedule configured for "${connector}". Nothing to remove.`); - process.exit(1); - } - delete config.schedule; - saveCfg(connector, config); - console.log(`✓ Schedule removed for ${connector}`); - }); - -// Registry commands -registerRegistryCommands(program); - -program.parse(); +export { program }; diff --git a/src/cli/options.ts b/src/cli/options.ts new file mode 100644 index 00000000..663c6dce --- /dev/null +++ b/src/cli/options.ts @@ -0,0 +1,76 @@ +/** Option helpers shared by commands: the document filter flags and value parsers. */ +import type { Command } from "commander"; +import { SOURCE_TYPES } from "../core/indexing.js"; + +/** "a, b,c" -> ["a", "b", "c"] (empty entries dropped). */ +export function splitList(value: string): string[] { + return value + .split(",") + .map((v) => v.trim()) + .filter((v) => v.length > 0); +} + +/** Flags added by addDocumentOptions, as commander parses them. */ +export interface DocumentFlags { + topic?: string | undefined; + library?: string | undefined; + libVersion?: string | undefined; + sourceType?: string | undefined; + tags?: string | undefined; + limit?: number | undefined; +} + +export interface DocumentOptionsConfig { + /** "filter" (default): flags select documents. "assign": flags set new documents' metadata. */ + mode?: "filter" | "assign"; + /** false: no -n/--limit. A string replaces its help text. */ + limit?: false | string; + /** Add --lib-version (default true). */ + version?: boolean; +} + +/** + * Add the shared document flags: --topic --library --lib-version --source-type --tags and + * -n/--limit. Map them to operation input with documentInput(). + */ +export function addDocumentOptions(cmd: Command, config: DocumentOptionsConfig = {}): Command { + const assign = config.mode === "assign"; + cmd + .option("--topic <topic>", assign ? "Topic (ID or name)" : "Only this topic (ID or name)") + .option("--library <name>", assign ? "Library name" : "Only this library"); + if (config.version !== false) { + cmd.option("--lib-version <version>", assign ? "Library version" : "Only this library version"); + } + cmd + .option( + "--source-type <type>", + `${assign ? "Source type" : "Only this source type"} (${SOURCE_TYPES.join(", ")})`, + ) + .option( + "--tags <tags>", + assign + ? "Tags to add (comma-separated)" + : "Only documents with all these tags (comma-separated)", + ); + if (config.limit !== false) { + cmd.option("-n, --limit <n>", config.limit ?? "Maximum results", Number); + } + return cmd; +} + +/** Operation input for the shared document flags (unset flags are left out). */ +export function documentInput(flags: DocumentFlags): Record<string, unknown> { + return defined({ + topic: flags.topic, + library: flags.library, + version: flags.libVersion, + sourceType: flags.sourceType, + tags: flags.tags === undefined ? undefined : splitList(flags.tags), + limit: flags.limit, + }); +} + +/** `obj` without undefined values (so operation defaults apply). */ +export function defined(obj: Record<string, unknown>): Record<string, unknown> { + return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined)); +} diff --git a/src/cli/repl.ts b/src/cli/repl.ts index c13cd706..03cc56cb 100644 --- a/src/cli/repl.ts +++ b/src/cli/repl.ts @@ -1,82 +1,51 @@ import * as readline from "node:readline/promises"; import { stdin as input, stdout as output } from "node:process"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { searchDocuments } from "../core/search.js"; -export interface ReplOptions { - db: Database.Database; - provider: EmbeddingProvider; - limit?: number; - /** Overridable for testing */ +export interface InteractiveOptions { + /** Called for each non-empty line the user enters. Errors are printed and the loop continues. */ + onQuery: (query: string) => Promise<void>; + /** Overridable for testing. */ createInterface?: () => readline.Interface; } -function formatResults( - results: { - title: string; - score: number; - library: string | null; - url: string | null; - content: string; - }[], - totalCount: number, -): string { - if (results.length === 0) { - return "No results found."; - } - - const lines: string[] = []; - lines.push(`\nShowing ${results.length} of ${totalCount} results:\n`); - - for (const r of results) { - lines.push(`\n── ${r.title} (score: ${r.score.toFixed(2)}) ──`); - if (r.library) lines.push(` Library: ${r.library}`); - if (r.url) lines.push(` Source: ${r.url}`); - lines.push(` ${r.content.slice(0, 200)}${r.content.length > 200 ? "..." : ""}`); - } - - return lines.join("\n"); -} - -export async function startRepl(options: ReplOptions): Promise<void> { - const { db, provider, limit = 5 } = options; - +/** Interactive search loop (`libscope search` with no query on a terminal). */ +export async function startInteractiveSearch(options: InteractiveOptions): Promise<void> { const rl = options.createInterface ? options.createInterface() : readline.createInterface({ input, output }); - console.log("LibScope interactive search (type 'quit' or 'exit' to leave)\n"); + console.log("LibScope interactive search (type 'quit' or 'exit', or press Ctrl+D, to leave)\n"); try { - for (;;) { - let query: string; - try { - query = await rl.question("search> "); - } catch { - // Ctrl+C or closed stream - break; - } + await promptLoop(rl, options.onQuery); + } finally { + rl.close(); + } +} - const trimmed = query.trim(); - if (!trimmed) continue; - if (trimmed === "quit" || trimmed === "exit") break; +/** Ask for one query, run it, then ask again until the user leaves. */ +async function promptLoop( + rl: readline.Interface, + onQuery: InteractiveOptions["onQuery"], +): Promise<void> { + let query: string; + try { + query = await rl.question("search> "); + } catch { + // Ctrl+C / Ctrl+D or closed stream + return; + } - try { - const { results, totalCount } = await searchDocuments(db, provider, { - query: trimmed, - limit, - }); - console.log(formatResults(results, totalCount)); - } catch (err) { - console.error(`Search error: ${err instanceof Error ? err.message : String(err)}`); - } + const trimmed = query.trim(); + if (trimmed === "quit" || trimmed === "exit") return; - console.log(); // blank line between searches + if (trimmed) { + try { + await onQuery(trimmed); + } catch (err) { + console.error(`✗ ${err instanceof Error ? err.message : String(err)}`); } - } finally { - rl.close(); + console.log(); } - - console.log("Goodbye!"); + return promptLoop(rl, onQuery); } diff --git a/src/cli/reporter.ts b/src/cli/reporter.ts index 08d5e558..637769d1 100644 --- a/src/cli/reporter.ts +++ b/src/cli/reporter.ts @@ -1,99 +1,47 @@ /** - * CLI output reporter — pretty human-readable output for interactive terminals. - * In verbose/JSON mode, a SilentReporter is used so pino JSON logs handle output. + * Progress line for long-running operations. Written to stderr so `--json` output on stdout + * stays clean, and only on a terminal (nothing is written when stderr is redirected). */ +import type { ProgressEvent } from "../core/operations/index.js"; -const RESET = "\x1b[0m"; -const GREEN = "\x1b[32m"; -const YELLOW = "\x1b[33m"; -const RED = "\x1b[31m"; -const CYAN = "\x1b[36m"; -const DIM = "\x1b[2m"; - -export interface CliReporter { - log(msg: string): void; - success(msg: string): void; - warn(msg: string): void; - error(msg: string): void; - progress(current: number, total: number, label: string): void; - clearProgress(): void; -} - -function buildBar(pct: number, width = 20): string { - const filled = Math.round((pct / 100) * width); - return "\u2588".repeat(filled) + "\u2591".repeat(width - filled); +export interface ProgressLine { + update(event: ProgressEvent): void; + clear(): void; } -/** Pretty human-readable reporter. Uses ANSI colors and \r progress lines. */ -class PrettyReporter implements CliReporter { - private hasProgress = false; - - log(msg: string): void { - this.clearProgress(); - process.stdout.write(`${msg}\n`); - } - - success(msg: string): void { - this.clearProgress(); - process.stdout.write(`${GREEN}\u2713${RESET} ${msg}\n`); - } - - warn(msg: string): void { - this.clearProgress(); - process.stderr.write(`${YELLOW}\u26a0${RESET} ${msg}\n`); - } - - error(msg: string): void { - this.clearProgress(); - process.stderr.write(`${RED}\u2717${RESET} ${msg}\n`); - } - - progress(current: number, total: number, label: string): void { - const pct = total > 0 ? Math.round((current / total) * 100) : 0; - const bar = buildBar(pct); - const truncatedLabel = label.length > 40 ? `${label.slice(0, 37)}...` : label; - const line = `${CYAN}[${bar}]${RESET} ${pct}% (${current}/${total}) ${DIM}${truncatedLabel}${RESET}`; - process.stdout.write(`\r${line}`); - this.hasProgress = true; - } - - clearProgress(): void { - if (this.hasProgress) { - const width = process.stdout.columns ?? 80; - process.stdout.write(`\r${" ".repeat(width - 1)}\r`); - this.hasProgress = false; - } - } +function bar(done: number, total: number, width = 20): string { + const filled = Math.min(width, Math.round((done / total) * width)); + return "█".repeat(filled) + "░".repeat(width - filled); } -/** No-op reporter: used in verbose/JSON mode where pino logs handle output. */ -class SilentReporter implements CliReporter { - log(_msg: string): void { - // intentionally empty — silent reporter - } - success(_msg: string): void { - // intentionally empty — silent reporter - } - warn(_msg: string): void { - // intentionally empty — silent reporter +/** Text of one progress line, e.g. "[████░░…] 40% (4/10) docs/a.md". */ +export function formatProgress(event: ProgressEvent): string { + const label = event.message ?? ""; + const shortLabel = label.length > 50 ? `...${label.slice(-47)}` : label; + if (event.total !== undefined && event.total > 0) { + const pct = Math.round((event.done / event.total) * 100); + return `[${bar(event.done, event.total)}] ${pct}% (${event.done}/${event.total}) ${shortLabel}`; } - error(_msg: string): void { - // intentionally empty — silent reporter - } - progress(_current: number, _total: number, _label: string): void { - // intentionally empty — silent reporter - } - clearProgress(): void { - // intentionally empty — silent reporter - } -} - -/** Returns true if verbose mode is active (flag or env var). */ -export function isVerbose(verbose?: boolean): boolean { - return verbose === true || process.env["LIBSCOPE_VERBOSE"] === "1"; + return `(${event.done}) ${shortLabel}`; } -/** Create a reporter appropriate for the current mode. */ -export function createReporter(verbose?: boolean): CliReporter { - return isVerbose(verbose) ? new SilentReporter() : new PrettyReporter(); +export function createProgressLine(stream: NodeJS.WriteStream = process.stderr): ProgressLine { + let shown = false; + const width = (): number => stream.columns ?? 80; + return { + update(event: ProgressEvent): void { + if (!stream.isTTY) return; + stream.write( + `\r${formatProgress(event) + .slice(0, width() - 1) + .padEnd(width() - 1)}`, + ); + shown = true; + }, + clear(): void { + if (!shown) return; + stream.write(`\r${" ".repeat(width() - 1)}\r`); + shown = false; + }, + }; } diff --git a/src/cli/run.ts b/src/cli/run.ts new file mode 100644 index 00000000..8716f7d8 --- /dev/null +++ b/src/cli/run.ts @@ -0,0 +1,73 @@ +/** + * Run an operation for a command and print its result: JSON with the global --json flag, + * otherwise the command's human formatter. + */ +import type { z } from "zod"; +import { runOperation, type Operation } from "../core/operations/index.js"; +import { getContext, getGlobalOptions, type ContextOptions } from "./context.js"; +import { createProgressLine } from "./reporter.js"; + +/** Run `op` with `input` in the command's context, showing progress on a terminal. */ +export async function call<S extends z.ZodObject, O>( + op: Operation<S, O>, + input: unknown, + options: ContextOptions = {}, +): Promise<O> { + const ctx = getContext(options); + const progress = createProgressLine(); + try { + return await runOperation(op, { ...ctx, onProgress: (p) => progress.update(p) }, input); + } finally { + progress.clear(); + } +} + +export function isJsonOutput(): boolean { + return getGlobalOptions().json === true; +} + +/** Print `result` as JSON (--json) or with `human`. */ +export function print<T>(result: T, human: (result: T) => void): void { + if (isJsonOutput()) { + console.log(JSON.stringify(result, null, 2)); + } else { + human(result); + } +} + +/** call() then print(). */ +export async function run<S extends z.ZodObject, O>( + op: Operation<S, O>, + input: unknown, + human: (result: O) => void, + options: ContextOptions = {}, +): Promise<O> { + const result = await call(op, input, options); + print(result, human); + return result; +} + +/** First `max` characters of `text` on one line, with "..." when cut. */ +export function preview(text: string, max = 200): string { + const flat = text.replaceAll("\n", " ").trim(); + return flat.length > max ? `${flat.slice(0, max)}...` : flat; +} + +/** "library 1.2" / "library". */ +export function libraryLabel(library: string, version: string | null | undefined): string { + return version ? `${library} ${version}` : library; +} + +/** "1 document" / "3 documents". */ +export function plural(count: number, noun: string): string { + return `${count} ${noun}${count === 1 ? "" : "s"}`; +} + +/** Print "message" when `items` is empty, else print each item. */ +export function printList<T>(items: readonly T[], empty: string, each: (item: T) => void): void { + if (items.length === 0) { + console.log(empty); + return; + } + for (const item of items) each(item); +} diff --git a/src/config-schema.ts b/src/config-schema.ts new file mode 100644 index 00000000..e5a5badb --- /dev/null +++ b/src/config-schema.ts @@ -0,0 +1,281 @@ +/** + * The configuration schema: the single source for config keys, defaults, validation, + * environment variable names, secret flags, and descriptions. + * + * Every leaf key `section.field` can be set from the environment variable + * `LIBSCOPE_<SECTION>_<FIELD>` (SCREAMING_SNAKE case), followed by any extra names in its + * metadata. Secret keys are read only from env vars and ~/.libscope/secrets.json. + */ +import { z } from "zod"; + +/** Metadata attached to each leaf of ConfigSchema. */ +export interface ConfigKeyMeta { + /** Extra env var names, checked in order after the canonical LIBSCOPE_<SECTION>_<FIELD>. */ + env?: readonly string[]; + /** Secret values are masked on display and stored in secrets.json, never in config.json. */ + secret?: boolean; + /** Default text for the docs table when the default depends on other keys. */ + defaultText?: string; +} + +/** Registry that holds the metadata of every leaf schema. */ +export const configKeyMeta = z.registry<ConfigKeyMeta>(); + +function leaf<T extends z.ZodType>(schema: T, meta: ConfigKeyMeta = {}): T { + configKeyMeta.add(schema, meta); + return schema; +} + +export const EMBEDDING_PROVIDERS = ["local", "ollama", "openai"] as const; +export const LLM_PROVIDERS = ["auto", "openai", "anthropic", "ollama", "passthrough"] as const; +export const LOG_LEVELS = ["debug", "info", "warn", "error", "silent"] as const; +/** Values of `mcp.toolsets`. "core" is always on; "all" enables every optional toolset. */ +export const MCP_TOOLSET_SETTINGS = ["core", "admin", "all"] as const; + +/** Ollama server URL used when embedding.url and llm.url are not set. */ +export const DEFAULT_OLLAMA_URL = "http://localhost:11434"; + +/** Largest accepted `embedding.dimensions` value. */ +const MAX_DIMENSIONS = 10000; + +const nonEmpty = (): z.ZodString => z.string().trim().min(1); + +export const ConfigSchema = z.object({ + embedding: z.object({ + provider: leaf(z.enum(EMBEDDING_PROVIDERS).default("local").describe("Embedding provider.")), + model: leaf( + nonEmpty() + .optional() + .describe( + "Embedding model. Ignored by the local provider (always Xenova/all-MiniLM-L6-v2).", + ), + { defaultText: "ollama: nomic-embed-text; openai: text-embedding-3-small" }, + ), + url: leaf( + nonEmpty() + .optional() + .describe("Server URL of the embedding provider. Used by the ollama provider."), + { defaultText: DEFAULT_OLLAMA_URL }, + ), + dimensions: leaf( + z + .number() + .int() + .positive() + .max(MAX_DIMENSIONS) + .optional() + .describe("Vector size, for models whose size is not built in."), + { defaultText: "size of the model" }, + ), + }), + llm: z.object({ + provider: leaf( + z + .enum(LLM_PROVIDERS) + .default("auto") + .describe( + 'LLM for "ask". "auto": passthrough under MCP, else openai if an OpenAI key is set, ' + + "else anthropic if an Anthropic key is set, else ollama if llm.url is set or " + + "embedding.provider is ollama, else none.", + ), + ), + model: leaf(nonEmpty().optional().describe("LLM model."), { + defaultText: "openai: gpt-4o-mini; anthropic: claude-3-5-haiku-20241022; ollama: llama3.2", + }), + url: leaf( + nonEmpty().optional().describe("Server URL of the LLM. Used by the ollama provider."), + { defaultText: `embedding.url, then ${DEFAULT_OLLAMA_URL}` }, + ), + }), + openai: z.object({ + apiKey: leaf(nonEmpty().optional().describe("OpenAI API key (embeddings and LLM)."), { + secret: true, + env: ["OPENAI_API_KEY"], + }), + }), + anthropic: z.object({ + apiKey: leaf(nonEmpty().optional().describe("Anthropic API key (LLM)."), { + secret: true, + env: ["ANTHROPIC_API_KEY"], + }), + }), + database: z.object({ + path: leaf( + nonEmpty() + .optional() + .describe("SQLite database file. A leading ~ is expanded. Unset: the workspace database."), + { defaultText: "~/.libscope/workspaces/<workspace>/libscope.db" }, + ), + }), + indexing: z.object({ + maxDocumentSize: leaf( + z + .number() + .int() + .positive() + .default(100 * 1024 * 1024) + .describe("Largest document to index, in bytes."), + ), + allowPrivateUrls: leaf( + z.boolean().default(false).describe("Allow fetching URLs on private or local networks."), + ), + allowSelfSignedCerts: leaf( + z.boolean().default(false).describe("Accept self-signed TLS certificates when fetching."), + ), + }), + logging: z.object({ + level: leaf(z.enum(LOG_LEVELS).default("info").describe("Log level.")), + }), + mcp: z.object({ + toolsets: leaf( + z + .array(z.enum(MCP_TOOLSET_SETTINGS)) + .default([]) + .describe( + "Optional MCP toolsets (comma list). admin adds sync, install-pack, list-packs and " + + "reindex-documents; all enables every optional toolset. The core tools are always on.", + ), + ), + }), +}); + +type SchemaOutput = z.output<typeof ConfigSchema>; +type SectionName = keyof SchemaOutput; + +/** Every settable key, as `section.field`, derived from ConfigSchema. */ +export type ConfigKey = { + [S in SectionName]: `${S}.${keyof SchemaOutput[S] & string}`; +}[SectionName]; + +/** One leaf of the schema, with everything needed to read, write, and document it. */ +export interface ConfigKeySpec { + key: ConfigKey; + section: SectionName; + field: string; + schema: z.ZodType; + /** Env var names in priority order (canonical name first). */ + env: readonly string[]; + secret: boolean; + description: string; + /** Default value, or undefined when the key has no fixed default. */ + defaultValue: unknown; + defaultText: string | undefined; + /** "string", "integer", "boolean", "list" (of `listValues`), or the enum values. */ + type: "string" | "integer" | "boolean" | "list" | readonly string[]; + /** Allowed items of a "list" key. */ + listValues?: readonly string[] | undefined; +} + +/** `indexing.allowPrivateUrls` -> `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS`. */ +export function canonicalEnvName(section: string, field: string): string { + const snake = field.replaceAll(/([a-z0-9])([A-Z])/g, "$1_$2").toUpperCase(); + return `LIBSCOPE_${section.toUpperCase()}_${snake}`; +} + +/** Strip optional/default wrappers to reach the base schema. */ +function baseSchema(schema: z.ZodType): z.ZodType { + let current: z.ZodType = schema; + while (current instanceof z.ZodOptional || current instanceof z.ZodDefault) { + current = current.unwrap() as z.ZodType; + } + return current; +} + +function enumValues(schema: z.ZodType): string[] | undefined { + return schema instanceof z.ZodEnum ? schema.options.map(String) : undefined; +} + +function leafType(schema: z.ZodType): ConfigKeySpec["type"] { + const base = baseSchema(schema); + if (base instanceof z.ZodArray) return "list"; + const values = enumValues(base); + if (values) return values; + if (base instanceof z.ZodBoolean) return "boolean"; + if (base instanceof z.ZodNumber) return "integer"; + return "string"; +} + +function listItemValues(schema: z.ZodType): string[] | undefined { + const base = baseSchema(schema); + return base instanceof z.ZodArray ? enumValues(base.element as z.ZodType) : undefined; +} + +function buildSpecs(): ConfigKeySpec[] { + const specs: ConfigKeySpec[] = []; + for (const [section, sectionSchema] of Object.entries(ConfigSchema.shape)) { + for (const [field, schemaValue] of Object.entries(sectionSchema.shape)) { + const schema = schemaValue as z.ZodType; + const meta = configKeyMeta.get(schema) ?? {}; + const parsedDefault = schema.safeParse(undefined); + specs.push({ + key: `${section}.${field}` as ConfigKey, + section: section as SectionName, + field, + schema, + env: [canonicalEnvName(section, field), ...(meta.env ?? [])], + secret: meta.secret ?? false, + description: schema.description ?? "", + defaultValue: parsedDefault.success ? parsedDefault.data : undefined, + defaultText: meta.defaultText, + type: leafType(schema), + listValues: listItemValues(schema), + }); + } + } + return specs; +} + +/** All leaf specs, in schema order. */ +export const CONFIG_KEY_SPECS: readonly ConfigKeySpec[] = buildSpecs(); + +const SPECS_BY_KEY: ReadonlyMap<string, ConfigKeySpec> = new Map( + CONFIG_KEY_SPECS.map((spec) => [spec.key, spec]), +); + +/** All settable config keys. */ +export const CONFIG_KEY_NAMES: readonly ConfigKey[] = CONFIG_KEY_SPECS.map((s) => s.key); + +/** Spec for a key, or undefined. */ +export function getConfigKeySpec(key: string): ConfigKeySpec | undefined { + return SPECS_BY_KEY.get(key); +} + +/** One row of the generated configuration reference. */ +export interface ConfigKeyTableRow { + key: ConfigKey; + /** "string", "integer", "boolean", "list of a | b", or the allowed values joined with " | ". */ + type: string; + /** Default value as text ("" when none). */ + default: string; + /** Env var names in priority order. */ + env: string[]; + secret: boolean; + description: string; +} + +function defaultAsText(spec: ConfigKeySpec): string { + if (spec.defaultText !== undefined) return spec.defaultText; + const value = spec.defaultValue; + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + return String(value); + } + if (Array.isArray(value)) return value.length > 0 ? value.join(",") : "none"; + return ""; +} + +function typeAsText(spec: ConfigKeySpec): string { + if (spec.type === "list") return `list of ${(spec.listValues ?? ["string"]).join(" | ")}`; + return typeof spec.type === "string" ? spec.type : spec.type.join(" | "); +} + +/** The key table (for docs and `config` help): key, type, default, env vars, secret, description. */ +export function getConfigKeyTable(): ConfigKeyTableRow[] { + return CONFIG_KEY_SPECS.map((spec) => ({ + key: spec.key, + type: typeAsText(spec), + default: defaultAsText(spec), + env: [...spec.env], + secret: spec.secret, + description: spec.description, + })); +} diff --git a/src/config.ts b/src/config.ts index e5249874..8455766c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,26 +1,67 @@ -import { readFileSync, writeFileSync, existsSync, mkdirSync, accessSync, constants } from "node:fs"; +import { + readFileSync, + writeFileSync, + existsSync, + mkdirSync, + accessSync, + chmodSync, + constants, +} from "node:fs"; import { join, dirname } from "node:path"; import { homedir } from "node:os"; -import { ConfigError } from "./errors.js"; +import { ConfigError, ValidationError } from "./errors.js"; import { getLogger } from "./logger.js"; +import { + CONFIG_KEY_NAMES, + CONFIG_KEY_SPECS, + getConfigKeySpec, + type ConfigKey, + type ConfigKeySpec, + type EMBEDDING_PROVIDERS, + type LLM_PROVIDERS, + type LOG_LEVELS, + type MCP_TOOLSET_SETTINGS, +} from "./config-schema.js"; + +export { + ConfigSchema, + CONFIG_KEY_NAMES, + DEFAULT_OLLAMA_URL, + getConfigKeyTable, + type ConfigKey, + type ConfigKeyTableRow, +} from "./config-schema.js"; + +/** Value of `llm.provider`. "auto" picks a provider when the LLM is created. */ +export type LlmProviderSetting = (typeof LLM_PROVIDERS)[number]; export interface LibScopeConfig { embedding: { - provider: "local" | "ollama" | "openai" | (string & {}); - ollamaUrl?: string; - ollamaModel?: string; - openaiApiKey?: string; - openaiModel?: string; + provider: (typeof EMBEDDING_PROVIDERS)[number] | (string & {}); + /** Embedding model. Unset: the provider's default model. */ + model?: string | undefined; + /** Server URL of the provider (used by ollama). Unset: http://localhost:11434. */ + url?: string | undefined; + /** Vector size override for models whose dimension is not built in. */ + dimensions?: number | undefined; }; llm?: { - provider?: "openai" | "ollama" | "anthropic" | "passthrough"; - model?: string; - ollamaUrl?: string; - openaiApiKey?: string; - anthropicApiKey?: string; + /** Unset means "auto". */ + provider?: LlmProviderSetting | undefined; + model?: string | undefined; + /** Server URL of the LLM (used by ollama). Unset: embedding.url, then http://localhost:11434. */ + url?: string | undefined; }; + /** Read from env (LIBSCOPE_OPENAI_API_KEY, OPENAI_API_KEY) or ~/.libscope/secrets.json. */ + openai?: { apiKey?: string | undefined }; + /** Read from env (LIBSCOPE_ANTHROPIC_API_KEY, ANTHROPIC_API_KEY) or ~/.libscope/secrets.json. */ + anthropic?: { apiKey?: string | undefined }; database: { - path: string; + /** + * Explicit SQLite file. Unset by default: the active workspace's database + * (~/.libscope/workspaces/<name>/libscope.db) is used. A leading `~` is expanded. + */ + path?: string; }; indexing: { maxDocumentSize: number; @@ -28,112 +69,260 @@ export interface LibScopeConfig { allowSelfSignedCerts: boolean; }; logging: { - level: "debug" | "info" | "warn" | "error" | "silent"; + level: (typeof LOG_LEVELS)[number]; + }; + mcp?: { + /** Optional MCP toolsets to enable. Unset: none (core tools only). */ + toolsets?: McpToolsetSetting[] | undefined; }; } -const DEFAULT_CONFIG: LibScopeConfig = { - embedding: { - provider: "local", - ollamaUrl: "http://localhost:11434", - ollamaModel: "nomic-embed-text", - openaiModel: "text-embedding-3-small", - }, - database: { - path: join(homedir(), ".libscope", "libscope.db"), - }, - indexing: { - maxDocumentSize: 100 * 1024 * 1024, // 100MB - allowPrivateUrls: false, - allowSelfSignedCerts: false, - }, - logging: { - level: "info", - }, -}; +/** Value of one `mcp.toolsets` item. */ +export type McpToolsetSetting = (typeof MCP_TOOLSET_SETTINGS)[number]; + +/** Expand a leading `~` (or `~/`) to the user's home directory. */ +export function expandHomeDir(path: string): string { + if (path === "~") return homedir(); + if (path.startsWith("~/") || path.startsWith("~\\")) return join(homedir(), path.slice(2)); + return path; +} function getConfigDir(): string { return join(homedir(), ".libscope"); } -function getUserConfigPath(): string { +/** Path to the user config file (~/.libscope/config.json). */ +export function getUserConfigPath(): string { return join(getConfigDir(), "config.json"); } +/** Path to the secrets file (~/.libscope/secrets.json, mode 0600). */ +export function getSecretsPath(): string { + return join(getConfigDir(), "secrets.json"); +} + function getProjectConfigPath(): string { return join(process.cwd(), ".libscope.json"); } -function loadJsonFile(path: string): Partial<LibScopeConfig> { +type RawConfig = Record<string, unknown>; +type ConfigValue = string | number | boolean | string[]; +/** A config layer flattened to `section.field` keys. */ +type FlatLayer = Record<string, unknown>; + +function isPlainObject(value: unknown): value is RawConfig { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function readJsonObject(path: string): RawConfig { try { if (!existsSync(path)) return {}; - const content = readFileSync(path, "utf-8"); - return JSON.parse(content) as Partial<LibScopeConfig>; + const parsed: unknown = JSON.parse(readFileSync(path, "utf-8")); + if (!isPlainObject(parsed)) throw new Error("top-level value must be a JSON object"); + return parsed; } catch (err) { throw new ConfigError(`Failed to read config file: ${path}`, err); } } -function getEnvOverrides(): Partial<LibScopeConfig> { - const overrides: Partial<LibScopeConfig> = {}; - const provider = process.env["LIBSCOPE_EMBEDDING_PROVIDER"]; - const openaiKey = process.env["LIBSCOPE_OPENAI_API_KEY"]; - const ollamaUrl = process.env["LIBSCOPE_OLLAMA_URL"]; +/** Write a JSON file under ~/.libscope (directory 0700, file 0600). */ +function writePrivateJson(path: string, data: RawConfig): void { + const dir = getConfigDir(); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + } + writeFileSync(path, JSON.stringify(data, null, 2) + "\n", { encoding: "utf-8", mode: 0o600 }); + // writeFileSync's mode only applies when the file is created; tighten existing files too. + chmodSync(path, 0o600); + invalidateConfigCache(); +} + +/** + * Read ~/.libscope/config.json as raw JSON, including keys this module does not model + * (e.g. `registries`). This and writeRawUserConfig are the only reader/writer of that file. + */ +export function readRawUserConfig(): RawConfig { + return readJsonObject(getUserConfigPath()); +} + +/** Write ~/.libscope/config.json (directory 0700, file 0600). */ +export function writeRawUserConfig(config: RawConfig): void { + writePrivateJson(getUserConfigPath(), config); +} + +function splitKey(key: string): [section: string, field: string] { + const dot = key.indexOf("."); + return dot === -1 ? [key, ""] : [key.slice(0, dot), key.slice(dot + 1)]; +} + +/** Flatten `{ section: { field: v } }` to `{ "section.field": v }` (one level only). */ +function flatten(raw: RawConfig): FlatLayer { + const flat: FlatLayer = {}; + for (const [section, values] of Object.entries(raw)) { + if (!isPlainObject(values)) continue; + for (const [field, value] of Object.entries(values)) { + if (value !== undefined) flat[`${section}.${field}`] = value; + } + } + return flat; +} + +function rawSection(raw: RawConfig, section: string): RawConfig { + const existing = raw[section]; + if (isPlainObject(existing)) return existing; + const created: RawConfig = {}; + raw[section] = created; + return created; +} - if (provider === "local" || provider === "ollama" || provider === "openai") { - overrides.embedding = { ...DEFAULT_CONFIG.embedding, provider }; +/** Validate a key name; throws ValidationError listing the valid keys. */ +export function parseConfigKey(name: string): ConfigKey { + if (!getConfigKeySpec(name)) { + throw new ValidationError( + `Unknown config key: ${name}. Valid keys: ${CONFIG_KEY_NAMES.join(", ")}`, + ); + } + return name as ConfigKey; +} + +function specFor(name: string): ConfigKeySpec { + const spec = getConfigKeySpec(parseConfigKey(name)); + if (!spec) throw new ValidationError(`Unknown config key: ${name}`); + return spec; +} + +/** Run the key's schema; throws ValidationError with the first issue. */ +function validateWithSchema(spec: ConfigKeySpec, value: unknown): ConfigValue { + const result = spec.schema.safeParse(value); + if (!result.success) { + const issue = result.error.issues[0]?.message ?? "invalid value"; + throw new ValidationError(`${spec.key}: ${issue} (got ${JSON.stringify(value)})`); + } + return result.data as ConfigValue; +} + +/** Convert a string (CLI argument or env var) to the key's type and validate it. */ +export function coerceConfigValue(key: ConfigKey, raw: string): ConfigValue { + const spec = specFor(key); + const { type } = spec; + if (type === "boolean") { + const lower = raw.trim().toLowerCase(); + if (lower === "true" || lower === "1") return true; + if (lower === "false" || lower === "0") return false; + throw new ValidationError(`${key} must be true or false (got "${raw}")`); + } + if (type === "integer") { + const n = Number(raw); + if (raw.trim() === "" || !Number.isInteger(n) || n <= 0) { + throw new ValidationError(`${key} must be a positive integer (got "${raw}")`); + } + return validateWithSchema(spec, n); + } + if (type === "string") { + if (raw.trim() === "") throw new ValidationError(`${key} must not be empty`); + return validateWithSchema(spec, raw); + } + if (type === "list") { + const items = raw + .split(",") + .map((item) => item.trim()) + .filter((item) => item !== ""); + return validateWithSchema(spec, items); } - if (openaiKey) { - overrides.embedding = { - ...(overrides.embedding ?? DEFAULT_CONFIG.embedding), - openaiApiKey: openaiKey, - }; + if (!type.includes(raw)) { + throw new ValidationError(`${key} must be one of: ${type.join(", ")} (got "${raw}")`); } - if (ollamaUrl) { - overrides.embedding = { ...(overrides.embedding ?? DEFAULT_CONFIG.embedding), ollamaUrl }; + return raw; +} + +function warnIgnored(what: string, err: unknown): void { + const reason = err instanceof Error ? err.message : String(err); + getLogger().warn(`Ignoring ${what}: ${reason}`); +} + +/** + * Validate a value read from a config file. Invalid values are logged and ignored, so one bad + * key does not stop libscope from starting. Strings are accepted for booleans, integers and + * lists (comma-separated). + */ +function readFileValue(spec: ConfigKeySpec, value: unknown, source: string): unknown { + try { + if ( + typeof value === "string" && + (spec.type === "boolean" || spec.type === "integer" || spec.type === "list") + ) { + return coerceConfigValue(spec.key, value); + } + return validateWithSchema(spec, value); + } catch (err) { + warnIgnored(`invalid config value in ${source}`, err); + return undefined; } +} - const llmProvider = process.env["LIBSCOPE_LLM_PROVIDER"]; - const llmModel = process.env["LIBSCOPE_LLM_MODEL"]; - const allowPrivate = process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"]; - const allowSelfSigned = process.env["LIBSCOPE_ALLOW_SELF_SIGNED_CERTS"]; - - if ( - allowPrivate === "true" || - allowPrivate === "1" || - allowSelfSigned === "true" || - allowSelfSigned === "1" - ) { - overrides.indexing = { - ...DEFAULT_CONFIG.indexing, - ...(allowPrivate === "true" || allowPrivate === "1" ? { allowPrivateUrls: true } : {}), - ...(allowSelfSigned === "true" || allowSelfSigned === "1" - ? { allowSelfSignedCerts: true } - : {}), - }; +/** + * Read a config file into a flat layer of known keys. `secrets` selects which keys are kept: + * secret keys (secrets.json) or non-secret keys (config files). Secret keys found in a config + * file are reported and ignored. + */ +function readFileLayer(path: string, secrets: boolean): FlatLayer { + const layer: FlatLayer = {}; + const ignoredSecrets: string[] = []; + for (const [key, value] of Object.entries(flatten(readJsonObject(path)))) { + const spec = getConfigKeySpec(key); + if (!spec) continue; + if (spec.secret !== secrets) { + if (spec.secret) ignoredSecrets.push(key); + continue; + } + const parsed = readFileValue(spec, value, path); + if (parsed !== undefined) layer[key] = parsed; } + if (ignoredSecrets.length > 0) { + getLogger().warn( + `Ignoring ${ignoredSecrets.join(", ")} in ${path}: API keys are read only from ` + + `environment variables and ${getSecretsPath()}. Run "libscope config set <key> <value>" to store them there.`, + ); + } + return layer; +} - if ( - llmProvider === "openai" || - llmProvider === "ollama" || - llmProvider === "anthropic" || - llmProvider === "passthrough" || - llmModel - ) { - const anthropicKey = process.env["LIBSCOPE_ANTHROPIC_API_KEY"]; - overrides.llm = { - ...(llmProvider === "openai" || - llmProvider === "ollama" || - llmProvider === "anthropic" || - llmProvider === "passthrough" - ? { provider: llmProvider } - : {}), - ...(llmModel ? { model: llmModel } : {}), - ...(anthropicKey ? { anthropicApiKey: anthropicKey } : {}), - }; +/** + * Collect config values from env vars. Only variables that are set appear in the result. + * Each key reads LIBSCOPE_<SECTION>_<FIELD>, then its extra names (e.g. OPENAI_API_KEY). + */ +function readEnvLayer(): FlatLayer { + const layer: FlatLayer = {}; + for (const spec of CONFIG_KEY_SPECS) { + const name = spec.env.find((n) => process.env[n] !== undefined && process.env[n] !== ""); + const value = name === undefined ? undefined : process.env[name]; + if (name === undefined || value === undefined) continue; + try { + layer[spec.key] = coerceConfigValue(spec.key, value); + } catch (err) { + warnIgnored(`env var ${name}`, err); + } } + return layer; +} + +/** Mask a secret for display: `sk-…abcd` for long values, `****` otherwise. */ +export function maskSecret(value: string): string { + return value.length >= 12 ? `${value.slice(0, 3)}…${value.slice(-4)}` : "****"; +} - return overrides; +/** Return a copy of the config with every secret masked (for display). */ +export function maskConfigSecrets(config: LibScopeConfig): LibScopeConfig { + const masked: LibScopeConfig = structuredClone(config); + const sections = masked as unknown as Record<string, RawConfig | undefined>; + for (const spec of CONFIG_KEY_SPECS) { + const section = sections[spec.section]; + const value = section?.[spec.field]; + if (section && spec.secret && typeof value === "string") { + section[spec.field] = maskSecret(value); + } + } + return masked; } let _configCache: LibScopeConfig | null = null; @@ -146,91 +335,74 @@ export function invalidateConfigCache(): void { _configCacheAt = 0; } -/** Load config with precedence: env > project > user > defaults. Result is cached for 30 s. */ +/** + * Load config. Precedence: env > project .libscope.json > ~/.libscope/config.json > schema + * defaults. Secret keys (API keys): env > ~/.libscope/secrets.json. No network calls are + * made. The result is cached for 30 s. + */ export function loadConfig(): LibScopeConfig { const now = Date.now(); if (_configCache && now - _configCacheAt < CONFIG_CACHE_TTL_MS) { return _configCache; } - const userConfig = loadJsonFile(getUserConfigPath()); - const projectConfig = loadJsonFile(getProjectConfigPath()); - const envOverrides = getEnvOverrides(); - - if ( - userConfig.embedding?.openaiApiKey || - userConfig.llm?.openaiApiKey || - userConfig.llm?.anthropicApiKey - ) { - getLogger().warn( - "API keys found in config file (~/.libscope/config.json). " + - "This is deprecated — please use environment variables (OPENAI_API_KEY, ANTHROPIC_API_KEY) instead. " + - "Keys in the config file will no longer be written back after the next save.", - ); + + const merged: FlatLayer = {}; + for (const spec of CONFIG_KEY_SPECS) { + if (spec.defaultValue !== undefined) merged[spec.key] = spec.defaultValue; } + Object.assign( + merged, + readFileLayer(getUserConfigPath(), false), + readFileLayer(getProjectConfigPath(), false), + readFileLayer(getSecretsPath(), true), + readEnvLayer(), + ); - const config: LibScopeConfig = { - embedding: { - ...DEFAULT_CONFIG.embedding, - ...userConfig.embedding, - ...projectConfig.embedding, - ...envOverrides.embedding, - }, - llm: { - ...userConfig.llm, - ...projectConfig.llm, - ...envOverrides.llm, - }, - database: { - ...DEFAULT_CONFIG.database, - ...userConfig.database, - ...projectConfig.database, - }, - indexing: { - ...DEFAULT_CONFIG.indexing, - ...userConfig.indexing, - ...projectConfig.indexing, - ...envOverrides.indexing, - }, - logging: { - ...DEFAULT_CONFIG.logging, - ...userConfig.logging, - ...projectConfig.logging, - }, - }; + const config: RawConfig = {}; + for (const [key, value] of Object.entries(merged)) { + const [section, field] = splitKey(key); + rawSection(config, section)[field] = value; + } + const dbPath = merged["database.path"]; + config["database"] = typeof dbPath === "string" ? { path: expandHomeDir(dbPath) } : {}; - validateConfig(config); + const result = config as unknown as LibScopeConfig; + validateConfig(result); - _configCache = config; + _configCache = result; _configCacheAt = now; - return config; + return result; } -/** Check embedding and LLM provider configuration for missing keys/URLs. */ +function apiKeyHint(provider: "OPENAI" | "ANTHROPIC"): string { + return ( + `Set LIBSCOPE_${provider}_API_KEY or ${provider}_API_KEY, ` + + `or run "libscope config set ${provider.toLowerCase()}.apiKey <key>".` + ); +} + +/** Check embedding and LLM provider configuration for missing keys. */ function validateProviderConfig(config: LibScopeConfig, warnings: string[]): void { - if (config.embedding.provider === "openai") { - const hasKey = config.embedding.openaiApiKey ?? process.env["OPENAI_API_KEY"]; - if (!hasKey) { - warnings.push( - 'embedding.provider is "openai" but no API key found. Set embedding.openaiApiKey or OPENAI_API_KEY env var.', - ); - } + const openaiKey = config.openai?.apiKey; + if (config.embedding.provider === "openai" && !openaiKey) { + warnings.push(`embedding.provider is "openai" but no API key found. ${apiKeyHint("OPENAI")}`); } - if (config.embedding.provider === "ollama" && !config.embedding.ollamaUrl) { - warnings.push('embedding.provider is "ollama" but embedding.ollamaUrl is not set.'); + if (config.embedding.provider === "local" && config.embedding.model) { + warnings.push( + `embedding.model "${config.embedding.model}" is ignored by the local provider (it always uses Xenova/all-MiniLM-L6-v2).`, + ); } - if (config.llm?.provider === "openai") { - const hasKey = - config.llm.openaiApiKey ?? config.embedding.openaiApiKey ?? process.env["OPENAI_API_KEY"]; - if (!hasKey) { - warnings.push( - 'llm.provider is "openai" but no API key found. Set llm.openaiApiKey or OPENAI_API_KEY env var.', - ); - } + if (config.llm?.provider === "openai" && !openaiKey) { + warnings.push(`llm.provider is "openai" but no API key found. ${apiKeyHint("OPENAI")}`); + } + if (config.llm?.provider === "anthropic" && !config.anthropic?.apiKey) { + warnings.push(`llm.provider is "anthropic" but no API key found. ${apiKeyHint("ANTHROPIC")}`); } } -/** Check that the database directory is writable or can be created. */ +/** Check that an explicit database directory is writable or can be created. */ function validateDatabasePath(config: LibScopeConfig, warnings: string[]): void { + if (!config.database.path) return; // workspace default — created on demand const dbDir = dirname(config.database.path); try { if (existsSync(dbDir)) { @@ -265,48 +437,68 @@ export function validateConfig(config: LibScopeConfig): string[] { return warnings; } -/** Save a config value to the user config file. */ -export function saveUserConfig(config: Partial<LibScopeConfig>): void { - const configDir = getConfigDir(); - if (!existsSync(configDir)) { - mkdirSync(configDir, { recursive: true }); - } - const existing = loadJsonFile(getUserConfigPath()); - const merged: LibScopeConfig = { - embedding: { - ...DEFAULT_CONFIG.embedding, - ...existing.embedding, - ...config.embedding, - }, - llm: { - ...existing.llm, - ...config.llm, - }, - database: { - ...DEFAULT_CONFIG.database, - ...existing.database, - ...config.database, - }, - indexing: { - ...DEFAULT_CONFIG.indexing, - ...existing.indexing, - ...config.indexing, - }, - logging: { - ...DEFAULT_CONFIG.logging, - ...existing.logging, - ...config.logging, - }, - }; +/** Surfaces that create an LLM provider. "auto" resolves to passthrough under MCP. */ +export type LlmSurface = "cli" | "mcp" | "api" | "sdk"; - // Security: never persist API keys to disk — use environment variables instead. - // Keys are read from env vars (OPENAI_API_KEY, ANTHROPIC_API_KEY) at runtime. - delete merged.embedding.openaiApiKey; - if (merged.llm) { - delete merged.llm.openaiApiKey; - delete merged.llm.anthropicApiKey; - } +/** LLM provider after "auto" is resolved. */ +export type ResolvedLlmProviderName = Exclude<LlmProviderSetting, "auto">; - writeFileSync(getUserConfigPath(), JSON.stringify(merged, null, 2), "utf-8"); - invalidateConfigCache(); +/** + * Resolve `llm.provider` ("auto" by default) to a concrete provider, or null when no LLM is + * available. Auto: passthrough under MCP; else openai if an OpenAI key is set; else anthropic + * if an Anthropic key is set; else ollama if llm.url is set or embedding.provider is "ollama"; + * else null. Reads only the config object; no network calls. + */ +export function resolveLlmProviderName( + config: LibScopeConfig, + options: { surface?: LlmSurface | undefined } = {}, +): ResolvedLlmProviderName | null { + const setting = config.llm?.provider ?? "auto"; + if (setting !== "auto") return setting; + if (options.surface === "mcp") return "passthrough"; + if (config.openai?.apiKey) return "openai"; + if (config.anthropic?.apiKey) return "anthropic"; + if (config.llm?.url || config.embedding.provider === "ollama") return "ollama"; + return null; +} + +/** The file a key is stored in: secrets.json for secret keys, else config.json. */ +export function getConfigFileFor(key: string): string { + return specFor(key).secret ? getSecretsPath() : getUserConfigPath(); +} + +/** + * Set one key in its file (config.json, or secrets.json for API keys). Validates and coerces + * the value and keeps every other key in the file (including `registries`). + */ +export function setUserConfigValue(key: string, rawValue: string): ConfigValue { + const spec = specFor(key); + const value = coerceConfigValue(spec.key, rawValue); + const path = getConfigFileFor(key); + const raw = readJsonObject(path); + rawSection(raw, spec.section)[spec.field] = value; + writePrivateJson(path, raw); + return value; +} + +/** Remove one key from its file. Returns false if the file did not set it. */ +export function unsetUserConfigValue(key: string): boolean { + const spec = specFor(key); + const path = getConfigFileFor(key); + const raw = readJsonObject(path); + const sectionObj = raw[spec.section]; + if (!isPlainObject(sectionObj) || !Object.hasOwn(sectionObj, spec.field)) return false; + delete sectionObj[spec.field]; + if (Object.keys(sectionObj).length === 0) delete raw[spec.section]; + writePrivateJson(path, raw); + return true; +} + +/** Read the effective value of a key from a loaded config. Secrets are masked. */ +export function getConfigValue(config: LibScopeConfig, key: string): ConfigValue | undefined { + const spec = specFor(key); + const sections = config as unknown as Record<string, RawConfig | undefined>; + const value = sections[spec.section]?.[spec.field] as ConfigValue | undefined; + if (typeof value === "string" && spec.secret) return maskSecret(value); + return value; } diff --git a/src/connectors/confluence.ts b/src/connectors/confluence.ts index 14e58997..800a249e 100644 --- a/src/connectors/confluence.ts +++ b/src/connectors/confluence.ts @@ -7,8 +7,10 @@ import { addTagsToDocument } from "../core/tags.js"; import { deleteDocument } from "../core/documents.js"; import { getLogger } from "../logger.js"; import { FetchError, ValidationError } from "../errors.js"; +import { forEachSequential } from "../utils/async.js"; +import { trimTrailingSlashes } from "../utils/strings.js"; import { fetchWithRetry } from "./http-utils.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; export interface ConfluenceConfig { baseUrl: string; @@ -112,17 +114,25 @@ async function confluenceFetch<T>(url: string, auth: string): Promise<T> { return (await response.json()) as T; } -async function fetchAllPages<T>(initialUrl: string, baseUrl: string, auth: string): Promise<T[]> { +async function fetchAllPages<T>( + initialUrl: string, + baseUrl: string, + auth: string, + signal?: AbortSignal, +): Promise<T[]> { const all: T[] = []; - let url: string | undefined = initialUrl; - while (url) { - const resp: PaginatedResponse<T> = await confluenceFetch<PaginatedResponse<T>>(url, auth); + // One page per call; follows the "next" link until there is none. + const fetchFrom = async (url: string | undefined): Promise<void> => { + if (!url) return; + signal?.throwIfAborted(); + const resp = await confluenceFetch<PaginatedResponse<T>>(url, auth); all.push(...resp.results); const next: string | undefined = resp._links?.next; - url = next ? `${baseUrl}${next}` : undefined; - } + return fetchFrom(next ? `${baseUrl}${next}` : undefined); + }; + await fetchFrom(initialUrl); return all; } @@ -137,11 +147,12 @@ function replaceStructuredMacros( ): string { const OPEN = "<ac:structured-macro"; const CLOSE = "</ac:structured-macro>"; + const lower = html.toLowerCase(); let result = ""; let pos = 0; while (pos < html.length) { - const start = html.toLowerCase().indexOf(OPEN.toLowerCase(), pos); + const start = lower.indexOf(OPEN, pos); if (start === -1) { result += html.slice(pos); break; @@ -152,7 +163,7 @@ function replaceStructuredMacros( break; } const attrs = html.slice(start, tagEnd + 1); - const closeStart = html.toLowerCase().indexOf(CLOSE.toLowerCase(), tagEnd + 1); + const closeStart = lower.indexOf(CLOSE, tagEnd + 1); if (closeStart === -1) { result += html.slice(pos); break; @@ -174,11 +185,14 @@ function replaceStructuredMacros( function replaceTagPairs(html: string, tagName: string, cb: (inner: string) => string): string { const openPrefix = `<${tagName}`; const closeTag = `</${tagName}>`; + const lower = html.toLowerCase(); + const lowerOpen = openPrefix.toLowerCase(); + const lowerClose = closeTag.toLowerCase(); let result = ""; let pos = 0; while (pos < html.length) { - const start = html.toLowerCase().indexOf(openPrefix.toLowerCase(), pos); + const start = lower.indexOf(lowerOpen, pos); if (start === -1) { result += html.slice(pos); break; @@ -188,7 +202,7 @@ function replaceTagPairs(html: string, tagName: string, cb: (inner: string) => s result += html.slice(pos); break; } - const closeStart = html.toLowerCase().indexOf(closeTag.toLowerCase(), tagEnd + 1); + const closeStart = lower.indexOf(lowerClose, tagEnd + 1); if (closeStart === -1) { result += html.slice(pos); break; @@ -204,10 +218,11 @@ function replaceTagPairs(html: string, tagName: string, cb: (inner: string) => s function extractTagContent(html: string, tagName: string): string { const open = `<${tagName}>`; const close = `</${tagName}>`; - const start = html.toLowerCase().indexOf(open.toLowerCase()); + const lower = html.toLowerCase(); + const start = lower.indexOf(open.toLowerCase()); if (start === -1) return ""; const contentStart = start + open.length; - const end = html.toLowerCase().indexOf(close.toLowerCase(), contentStart); + const end = lower.indexOf(close.toLowerCase(), contentStart); if (end === -1) return ""; return html.slice(contentStart, end); } @@ -218,11 +233,12 @@ function extractTagContent(html: string, tagName: string): string { */ function removeSelfClosingMacros(html: string, nameTest: RegExp): string { const OPEN = "<ac:structured-macro"; + const lower = html.toLowerCase(); let result = ""; let pos = 0; while (pos < html.length) { - const start = html.toLowerCase().indexOf(OPEN.toLowerCase(), pos); + const start = lower.indexOf(OPEN, pos); if (start === -1) { result += html.slice(pos); break; @@ -443,26 +459,29 @@ interface SyncConfluenceSpaceOptions { urls: ApiUrls; auth: string; result: ConfluenceSyncResult; + signal: AbortSignal | undefined; } /** Sync all pages within a single Confluence space. */ async function syncConfluenceSpace(options: SyncConfluenceSpaceOptions): Promise<void> { - const { db, provider, space, confluenceType, base, urls, auth, result } = options; + const { db, provider, space, confluenceType, base, urls, auth, result, signal } = options; const log = getLogger(); const topic = createTopic(db, { name: space.name }); let pages: ConfluencePage[]; try { const spaceRef = confluenceType === "server" ? space.key : space.id; - pages = await fetchAllPages<ConfluencePage>(urls.spacePages(spaceRef), base, auth); + pages = await fetchAllPages<ConfluencePage>(urls.spacePages(spaceRef), base, auth, signal); } catch (err) { + if (signal?.aborted) throw err; const msg = err instanceof Error ? err.message : String(err); log.error({ spaceKey: space.key, err }, "Failed to fetch pages for space"); result.errors.push({ page: `space:${space.key}`, error: msg }); return; } - for (const page of pages) { + await forEachSequential(pages, async (page) => { + signal?.throwIfAborted(); try { const outcome = await indexConfluencePage({ db, @@ -474,16 +493,24 @@ async function syncConfluenceSpace(options: SyncConfluenceSpaceOptions): Promise urls, auth, }); - if (outcome === "indexed") result.pagesIndexed++; - if (outcome === "updated") { - result.pagesUpdated++; - result.pagesIndexed++; - } + countPageOutcome(outcome, result); } catch (err) { const msg = err instanceof Error ? err.message : String(err); log.error({ pageId: page.id, title: page.title, err }, "Failed to index page"); result.errors.push({ page: page.title, error: msg }); } + }); +} + +/** Add one page's outcome to the sync counts (an updated page also counts as indexed). */ +function countPageOutcome( + outcome: Awaited<ReturnType<typeof indexConfluencePage>>, + result: ConfluenceSyncResult, +): void { + if (outcome === "indexed") result.pagesIndexed++; + if (outcome === "updated") { + result.pagesUpdated++; + result.pagesIndexed++; } } @@ -498,7 +525,7 @@ function validateConfluenceConfig(config: ConfluenceConfig): void { const confluenceType = config.type ?? "cloud"; if (confluenceType === "cloud" && !config.email?.trim()) { throw new ValidationError( - "Confluence email is required for Cloud. For Server/Data Center, use --type server", + "Confluence email is required for Cloud. For Server/Data Center, use --server", ); } } @@ -507,6 +534,27 @@ export async function syncConfluence( db: Database.Database, provider: EmbeddingProvider, config: ConfluenceConfig, + options: ConnectorSyncOptions = {}, +): Promise<ConfluenceSyncResult> { + return trackSync( + db, + "confluence", + options.syncName ?? trimTrailingSlashes(config.baseUrl), + () => runConfluenceSync(db, provider, config, options.signal), + (result) => ({ + added: result.pagesIndexed, + updated: result.pagesUpdated, + deleted: 0, + errored: result.errors.length, + }), + ); +} + +async function runConfluenceSync( + db: Database.Database, + provider: EmbeddingProvider, + config: ConfluenceConfig, + signal: AbortSignal | undefined, ): Promise<ConfluenceSyncResult> { const log = getLogger(); @@ -514,67 +562,54 @@ export async function syncConfluence( const confluenceType = config.type ?? "cloud"; const auth = buildAuthHeader(confluenceType, config.email, config.token); - let base = config.baseUrl; - while (base.endsWith("/")) base = base.slice(0, -1); + const base = trimTrailingSlashes(config.baseUrl); const urls = getApiUrls(base, confluenceType); - const syncId = startSync(db, "confluence", base); - try { - const result: ConfluenceSyncResult = { - spaces: 0, - pagesIndexed: 0, - pagesUpdated: 0, - errors: [], - }; - - log.info({ baseUrl: base }, "Starting Confluence sync"); - - const allSpaces = await fetchAllPages<ConfluenceSpace>(urls.spaces, base, auth); - const excludeSet = new Set(config.excludeSpaces ?? []); - const requestedAll = config.spaces.length === 1 && config.spaces[0] === "all"; - const spacesToSync = allSpaces.filter((s) => { - if (excludeSet.has(s.key)) return false; - return requestedAll || config.spaces.includes(s.key); - }); + const result: ConfluenceSyncResult = { + spaces: 0, + pagesIndexed: 0, + pagesUpdated: 0, + errors: [], + }; - result.spaces = spacesToSync.length; - log.info({ spaceCount: spacesToSync.length }, "Spaces to sync"); + log.info({ baseUrl: base }, "Starting Confluence sync"); - for (const space of spacesToSync) { - await syncConfluenceSpace({ - db, - provider, - space, - confluenceType, - base, - urls, - auth, - result, - }); - } + const allSpaces = await fetchAllPages<ConfluenceSpace>(urls.spaces, base, auth, signal); + const excludeSet = new Set(config.excludeSpaces ?? []); + const requestedAll = config.spaces.length === 1 && config.spaces[0] === "all"; + const spacesToSync = allSpaces.filter((s) => { + if (excludeSet.has(s.key)) return false; + return requestedAll || config.spaces.includes(s.key); + }); - log.info( - { - spaces: result.spaces, - indexed: result.pagesIndexed, - updated: result.pagesUpdated, - errors: result.errors.length, - }, - "Confluence sync complete", - ); + result.spaces = spacesToSync.length; + log.info({ spaceCount: spacesToSync.length }, "Spaces to sync"); + + await forEachSequential(spacesToSync, (space) => + syncConfluenceSpace({ + db, + provider, + space, + confluenceType, + base, + urls, + auth, + result, + signal, + }), + ); - completeSync(db, syncId, { - added: result.pagesIndexed, + log.info( + { + spaces: result.spaces, + indexed: result.pagesIndexed, updated: result.pagesUpdated, - deleted: 0, - errored: result.errors.length, - }); + errors: result.errors.length, + }, + "Confluence sync complete", + ); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; - } + return result; } export function disconnectConfluence(db: Database.Database): number { diff --git a/src/connectors/docs.ts b/src/connectors/docs.ts index b4671759..d56e7294 100644 --- a/src/connectors/docs.ts +++ b/src/connectors/docs.ts @@ -10,10 +10,11 @@ import { NodeHtmlMarkdown } from "node-html-markdown"; import { ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; import { fetchRaw } from "../core/url-fetcher.js"; +import { decodeHtmlEntities } from "../core/html-entities.js"; import type { FetchOptions } from "../core/url-fetcher.js"; import { indexDocument } from "../core/indexing.js"; -import { listDocuments, deleteDocument } from "../core/documents.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { deleteDocument } from "../core/documents.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; import type { EmbeddingProvider } from "../providers/embedding.js"; // Source type used to tag all docs-connector documents. @@ -354,7 +355,7 @@ export function extractDocTitle(html: string, url: string): string { .slice(innerStart, h1CloseIdx) .replaceAll(/<[^>]{1,2000}>/g, "") .trim(); - if (title) return title; + if (title) return decodeHtmlEntities(title); } } @@ -362,7 +363,7 @@ export function extractDocTitle(html: string, url: string): string { const titleTagMatch = /<title[^>]{0,2000}>([^<]+)<\/title>/i.exec(html); if (titleTagMatch?.[1]) { const title = titleTagMatch[1].trim(); - if (title) return title; + if (title) return decodeHtmlEntities(title); } // Last resort: derive from URL path @@ -495,8 +496,8 @@ interface PageContext { db: Database.Database; provider: EmbeddingProvider; config: DocSiteConfig; - /** Map of normalised URL → existing document ID for update detection. */ - existingUrlMap: Map<string, string>; + /** Normalised URLs of existing documents, for update detection. */ + existingUrls: Set<string>; result: DocSiteSyncResult; } @@ -520,7 +521,7 @@ async function processPage(url: string, html: string, ctx: PageContext): Promise } const normalised = normalizeUrl(url); - const isKnown = ctx.existingUrlMap.has(normalised); + const isKnown = ctx.existingUrls.has(normalised); const indexed = await indexDocument(ctx.db, ctx.provider, { title, @@ -576,6 +577,21 @@ function validateDocSiteConfig(config: DocSiteConfig): URL { return baseUrl; } +/** + * Normalised URLs of every docs-connector document (optionally scoped to one library). + * Reads only the url column and applies no row limit, so large sites are tracked fully. + */ +function loadExistingDocUrls(db: Database.Database, library: string | undefined): Set<string> { + let sql = "SELECT id, url FROM documents WHERE source_type = ? AND url IS NOT NULL"; + const params: string[] = [SOURCE_TYPE]; + if (library) { + sql += " AND library = ?"; + params.push(library); + } + const rows = db.prepare(sql).all(...params) as Array<{ id: string; url: string }>; + return new Set(rows.map((r) => normalizeUrl(r.url))); +} + /** Discover URLs via sitemap.xml and root page links, populating the BFS queue. */ async function discoverUrls( config: DocSiteConfig, @@ -613,10 +629,95 @@ async function discoverUrls( } } +interface CrawlEntry { + url: string; + depth: number; +} + +/** State shared by the pages of one documentation crawl. */ +interface CrawlState { + page: PageContext; + fetchOptions: FetchOptions; + visited: Set<string>; + queue: CrawlEntry[]; + maxPages: number; + maxDepth: number; + pathPrefix: string; +} + +/** Fetch and index one queued page, then queue its unvisited links within the depth budget. */ +async function crawlPage({ url, depth }: CrawlEntry, crawl: CrawlState): Promise<void> { + const { visited, queue, maxPages, maxDepth, pathPrefix } = crawl; + if (visited.size > maxPages) return; + + let html: string; + let contentType: string; + try { + const raw = await fetchRaw(url, crawl.fetchOptions); + html = raw.body; + contentType = raw.contentType; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + getLogger().warn({ url, error: msg }, "Failed to fetch documentation page"); + crawl.page.result.errors.push({ url, error: msg }); + return; + } + + // Only process HTML pages (skip binary/asset responses that slipped through) + if (!contentType.includes("text/html") && !contentType.includes("text/plain")) { + return; + } + + await processPage(url, html, crawl.page); + + // Continue link discovery if within depth budget + if (depth < maxDepth) { + for (const link of extractDocLinks(html, url, pathPrefix)) { + if (!visited.has(link)) { + visited.add(link); + queue.push({ url: link, depth: depth + 1 }); + } + } + } +} + +/** Crawl the queue in batches of `concurrency` pages until it is empty or maxPages is reached. */ +async function crawlQueue( + crawl: CrawlState, + options: { concurrency: number; signal: AbortSignal | undefined }, +): Promise<void> { + if (crawl.queue.length === 0 || crawl.visited.size > crawl.maxPages) return; + options.signal?.throwIfAborted(); + const batch = crawl.queue.splice(0, options.concurrency); + await Promise.allSettled(batch.map((entry) => crawlPage(entry, crawl))); + return crawlQueue(crawl, options); +} + export async function syncDocSite( db: Database.Database, provider: EmbeddingProvider, config: DocSiteConfig, + options: ConnectorSyncOptions = {}, +): Promise<DocSiteSyncResult> { + return trackSync( + db, + CONNECTOR_TYPE, + options.syncName ?? config.url, + () => runDocSiteSync(db, provider, config, options.signal), + (result) => ({ + added: result.pagesIndexed, + updated: result.pagesUpdated, + deleted: 0, + errored: result.errors.length, + }), + ); +} + +async function runDocSiteSync( + db: Database.Database, + provider: EmbeddingProvider, + config: DocSiteConfig, + signal: AbortSignal | undefined, ): Promise<DocSiteSyncResult> { const log = getLogger(); @@ -645,128 +746,75 @@ export async function syncDocSite( errors: [], }; - const syncId = startSync(db, CONNECTOR_TYPE, config.url); + // --- Fetch root page --- + log.info({ url: config.url }, "Fetching documentation root page"); + let rootHtml: string; try { - // --- Fetch root page --- - log.info({ url: config.url }, "Fetching documentation root page"); - - let rootHtml: string; - try { - const raw = await fetchRaw(config.url, fetchOptions); - rootHtml = raw.body; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - throw new Error(`Failed to fetch root page: ${msg}`); - } - - // --- Detect site type --- - result.detectedType = - config.type !== undefined && config.type !== "auto" - ? config.type - : detectDocSiteType(rootHtml); - - log.info({ type: result.detectedType, url: config.url }, "Documentation site type"); - - // --- URL discovery --- - const visited = new Set<string>(); - const queue: Array<{ url: string; depth: number }> = []; - const rootNormalised = normalizeUrl(config.url); - visited.add(rootNormalised); - - await discoverUrls(config, baseUrl, rootHtml, pathPrefix, fetchOptions, visited, queue); - - // --- Build existing-URL index for update tracking --- - const existingDocs = listDocuments(db, { sourceType: SOURCE_TYPE, library: config.library }); - const existingUrlMap = new Map<string, string>( - existingDocs - .filter((d): d is typeof d & { url: string } => d.url !== null) - .map((d) => [normalizeUrl(d.url), d.id]), - ); - - const ctx: PageContext = { - siteType: result.detectedType, - db, - provider, - config, - existingUrlMap, - result, - }; - - // --- Process the root page first --- - await processPage(rootNormalised, rootHtml, ctx); - - // --- BFS crawl --- - while (queue.length > 0 && visited.size <= maxPages) { - const batch = queue.splice(0, concurrency); - - await Promise.allSettled( - batch.map(async ({ url, depth }) => { - if (visited.size > maxPages) return; - - let html: string; - let contentType: string; - try { - const raw = await fetchRaw(url, fetchOptions); - html = raw.body; - contentType = raw.contentType; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - log.warn({ url, error: msg }, "Failed to fetch documentation page"); - result.errors.push({ url, error: msg }); - return; - } - - // Only process HTML pages (skip binary/asset responses that slipped through) - if (!contentType.includes("text/html") && !contentType.includes("text/plain")) { - return; - } - - await processPage(url, html, ctx); - - // Continue link discovery if within depth budget - if (depth < maxDepth) { - for (const link of extractDocLinks(html, url, pathPrefix)) { - if (!visited.has(link)) { - visited.add(link); - queue.push({ url: link, depth: depth + 1 }); - } - } - } - }), - ); - } - - completeSync(db, syncId, { - added: result.pagesIndexed, - updated: result.pagesUpdated, - deleted: 0, - errored: result.errors.length, - }); - - log.info( - { - pagesIndexed: result.pagesIndexed, - pagesUpdated: result.pagesUpdated, - pagesSkipped: result.pagesSkipped, - errors: result.errors.length, - }, - "Documentation site sync complete", - ); - - return result; + const raw = await fetchRaw(config.url, fetchOptions); + rootHtml = raw.body; } catch (err) { const msg = err instanceof Error ? err.message : String(err); - failSync(db, syncId, msg); - throw err; + throw new Error(`Failed to fetch root page: ${msg}`); } + + // --- Detect site type --- + result.detectedType = + config.type !== undefined && config.type !== "auto" ? config.type : detectDocSiteType(rootHtml); + + log.info({ type: result.detectedType, url: config.url }, "Documentation site type"); + + // --- URL discovery --- + const visited = new Set<string>(); + const queue: Array<{ url: string; depth: number }> = []; + const rootNormalised = normalizeUrl(config.url); + visited.add(rootNormalised); + + await discoverUrls(config, baseUrl, rootHtml, pathPrefix, fetchOptions, visited, queue); + + // --- Build existing-URL index for update tracking --- + const existingUrls = loadExistingDocUrls(db, config.library); + + const ctx: PageContext = { + siteType: result.detectedType, + db, + provider, + config, + existingUrls, + result, + }; + + // --- Process the root page first --- + await processPage(rootNormalised, rootHtml, ctx); + + // --- BFS crawl --- + await crawlQueue( + { page: ctx, fetchOptions, visited, queue, maxPages, maxDepth, pathPrefix }, + { + concurrency, + signal, + }, + ); + + log.info( + { + pagesIndexed: result.pagesIndexed, + pagesUpdated: result.pagesUpdated, + pagesSkipped: result.pagesSkipped, + errors: result.errors.length, + }, + "Documentation site sync complete", + ); + + return result; } /** * Remove all documents that were indexed from a given documentation site. * - * Identifies documents by URL prefix (`siteUrl + "%"`) so only pages that - * originated from the specified site are removed. + * Removes only the pages this connector indexed (source type "library", added by the crawler) + * whose URL starts with the site URL, so documents added another way under the same site + * (for example with `libscope add <url>`) are kept. * * @param db The database connection. * @param siteUrl Root URL of the documentation site (used as URL prefix filter). @@ -784,10 +832,13 @@ export function disconnectDocSite(db: Database.Database, siteUrl: string): numbe throw new ValidationError(`Invalid site URL for disconnect: ${siteUrl}`); } - // Parameterised LIKE — the prefix is derived from a validated URL, not user input. + // An exact prefix comparison: LIKE would treat "_" and "%" in the URL as wildcards. const rows = db - .prepare("SELECT id FROM documents WHERE url LIKE ?") - .all(`${basePrefix}%`) as Array<{ id: string }>; + .prepare( + `SELECT id FROM documents + WHERE source_type = ? AND submitted_by = 'crawler' AND substr(url, 1, ?) = ?`, + ) + .all(SOURCE_TYPE, basePrefix.length, basePrefix) as Array<{ id: string }>; let removed = 0; for (const row of rows) { diff --git a/src/connectors/http-utils.ts b/src/connectors/http-utils.ts index e8d10718..5002dbaf 100644 --- a/src/connectors/http-utils.ts +++ b/src/connectors/http-utils.ts @@ -1,14 +1,8 @@ -import { Agent } from "undici"; +import type { Agent } from "undici"; import { getLogger } from "../logger.js"; import { FetchError } from "../errors.js"; import { loadConfig } from "../config.js"; - -/** Lazy singleton undici Agent that skips TLS certificate verification. */ -let _insecureAgent: Agent | undefined; -function getInsecureAgent(): Agent { - _insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } }); - return _insecureAgent; -} +import { getInsecureAgent } from "../core/url-fetcher.js"; export interface RetryConfig { maxRetries?: number; @@ -60,7 +54,11 @@ export async function fetchWithRetry( const dispatcher = config.indexing.allowSelfSignedCerts ? getInsecureAgent() : undefined; const fetchOptions = buildFetchOptions(options, dispatcher); - for (let attempt = 0; attempt <= maxRetries; attempt++) { + const attemptFetch = async (attempt: number): Promise<Response> => { + // False only for a negative (or NaN) maxRetries: no attempt is allowed at all. + const allowed = attempt <= maxRetries; + if (!allowed) throw new FetchError("fetchWithRetry: unexpected code path"); + const response = await fetch(url, fetchOptions); if (!isRetryableStatus(response.status)) return response; @@ -76,8 +74,8 @@ export async function fetchWithRetry( "Retrying after transient error", ); await new Promise((resolve) => setTimeout(resolve, delayMs)); - } + return attemptFetch(attempt + 1); + }; - // Unreachable, but satisfies TypeScript - throw new FetchError("fetchWithRetry: unexpected code path"); + return attemptFetch(0); } diff --git a/src/connectors/index.ts b/src/connectors/index.ts index a35fdd59..ca5e5aa3 100644 --- a/src/connectors/index.ts +++ b/src/connectors/index.ts @@ -1,4 +1,12 @@ -import { readFileSync, writeFileSync, mkdirSync, existsSync, chmodSync } from "node:fs"; +import { + readFileSync, + readdirSync, + writeFileSync, + mkdirSync, + existsSync, + chmodSync, + unlinkSync, +} from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; import { ConfigError } from "../errors.js"; @@ -108,20 +116,30 @@ export function deleteDbConnectorConfig(db: Database.Database, type: string): bo return result.changes > 0; } -const CONNECTORS_DIR = join(homedir(), ".libscope", "connectors"); +/** Directory holding named connector configs (~/.libscope/connectors). */ +export function getConnectorsDir(): string { + return join(homedir(), ".libscope", "connectors"); +} -function ensureConnectorsDir(): void { - if (existsSync(CONNECTORS_DIR)) { +function ensureConnectorsDir(): string { + const dir = getConnectorsDir(); + if (existsSync(dir)) { // Remediate existing directories that may have permissive permissions - restrictPermissions(CONNECTORS_DIR, 0o700); + restrictPermissions(dir, 0o700); } else { - mkdirSync(CONNECTORS_DIR, { recursive: true, mode: 0o700 }); + mkdirSync(dir, { recursive: true, mode: 0o700 }); } try { - chmodSync(CONNECTORS_DIR, 0o700); + chmodSync(dir, 0o700); } catch { // chmod may fail in test environments or non-POSIX systems } + return dir; +} + +function namedConfigPath(name: string): string { + validateConnectorName(name); + return join(getConnectorsDir(), `${name}.json`); } function validateConnectorName(name: string): void { @@ -133,19 +151,17 @@ function validateConnectorName(name: string): void { /** Save a named connector config to ~/.libscope/connectors/<name>.json */ export function saveNamedConnectorConfig(name: string, config: object): void { validateConnectorName(name); - ensureConnectorsDir(); - const filePath = join(CONNECTORS_DIR, `${name}.json`); + const filePath = join(ensureConnectorsDir(), `${name}.json`); writeRestrictedFile(filePath, JSON.stringify(config, null, 2)); getLogger().info({ connector: name }, "Connector config saved"); } /** Load a named connector config from ~/.libscope/connectors/<name>.json */ export function loadNamedConnectorConfig<T>(name: string): T { - validateConnectorName(name); - const filePath = join(CONNECTORS_DIR, `${name}.json`); + const filePath = namedConfigPath(name); if (!existsSync(filePath)) { throw new ConfigError( - `No connector config found for "${name}". Run 'libscope connect ${name}' first.`, + `No connector config found for "${name}". Run 'libscope connect <type> --name ${name}' first.`, ); } const raw = readFileSync(filePath, "utf-8"); @@ -158,9 +174,39 @@ export function loadNamedConnectorConfig<T>(name: string): T { /** Check if a named connector config exists */ export function hasNamedConnectorConfig(name: string): boolean { - validateConnectorName(name); - const filePath = join(CONNECTORS_DIR, `${name}.json`); - return existsSync(filePath); + return existsSync(namedConfigPath(name)); +} + +/** + * Every named connector config in ~/.libscope/connectors, sorted by name. A file that cannot + * be read or parsed is skipped with a warning. + */ +export function listNamedConnectorConfigs(): Array<{ + name: string; + config: Record<string, unknown>; +}> { + const dir = getConnectorsDir(); + if (!existsSync(dir)) return []; + const entries: Array<{ name: string; config: Record<string, unknown> }> = []; + for (const file of readdirSync(dir).filter((f) => f.endsWith(".json"))) { + const name = file.slice(0, -".json".length); + try { + const config = JSON.parse(readFileSync(join(dir, file), "utf-8")) as Record<string, unknown>; + entries.push({ name, config }); + } catch (err) { + getLogger().warn({ file, err }, "Skipping unreadable connector config"); + } + } + return entries.sort((a, b) => a.name.localeCompare(b.name)); +} + +/** Delete a named connector config. Returns false when none existed. */ +export function deleteNamedConnectorConfig(name: string): boolean { + const filePath = namedConfigPath(name); + if (!existsSync(filePath)) return false; + unlinkSync(filePath); + getLogger().info({ connector: name }, "Connector config deleted"); + return true; } /** @@ -216,8 +262,9 @@ export { failSync, getConnectorStatus, getSyncHistory, + trackSync, } from "./sync-tracker.js"; -export type { SyncStats, ConnectorSyncRow } from "./sync-tracker.js"; +export type { SyncStats, ConnectorSyncRow, ConnectorSyncOptions } from "./sync-tracker.js"; export { syncNotion, convertNotionBlocks, disconnectNotion } from "./notion.js"; export type { NotionConfig, NotionSyncResult, NotionBlock } from "./notion.js"; diff --git a/src/connectors/notion.ts b/src/connectors/notion.ts index 2b1ede86..de8e72d0 100644 --- a/src/connectors/notion.ts +++ b/src/connectors/notion.ts @@ -2,10 +2,11 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { getLogger } from "../logger.js"; import { ValidationError, FetchError } from "../errors.js"; +import { forEachSequential } from "../utils/async.js"; import { fetchWithRetry } from "./http-utils.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; const NOTION_API_BASE = "https://api.notion.com"; const NOTION_VERSION = "2022-06-28"; @@ -106,7 +107,11 @@ async function notionFetch<T>( return (await response.json()) as T; } -async function searchNotion(token: string, lastSync?: string): Promise<NotionSearchResult[]> { +async function searchNotion( + token: string, + lastSync: string | undefined, + signal: AbortSignal | undefined, +): Promise<NotionSearchResult[]> { const log = getLogger(); const allResults: NotionSearchResult[] = []; let cursor: string | null = null; @@ -114,6 +119,7 @@ async function searchNotion(token: string, lastSync?: string): Promise<NotionSea const MAX_PAGES = 10_000; while (hasMore) { + signal?.throwIfAborted(); const body: Record<string, unknown> = { page_size: 100 }; if (cursor) body["start_cursor"] = cursor; if (lastSync) { @@ -181,12 +187,13 @@ async function fetchBlockChildren( } // Recursively fetch children - for (const block of allBlocks) { - if (block.has_children) { + await forEachSequential( + allBlocks.filter((block) => block.has_children), + async (block) => { const children = await fetchBlockChildren(token, block.id, depth + 1, maxDepth); (block as Record<string, unknown>)["children"] = children; - } - } + }, + ); return allBlocks; } @@ -460,13 +467,15 @@ async function syncNotionDatabase( token: string, item: NotionSearchResult, excludeSet: Set<string>, + signal: AbortSignal | undefined, ): Promise<void> { const log = getLogger(); const dbTitle = extractTitle(item); const rows = await queryDatabase(token, item.id); - for (const row of rows) { - if (excludeSet.has(row.id)) continue; + await forEachSequential(rows, async (row) => { + signal?.throwIfAborted(); + if (excludeSet.has(row.id)) return; const rowTitle = extractTitle(row); const tags = row.properties ? extractPropertyMetadata(row.properties) : []; @@ -477,7 +486,7 @@ async function syncNotionDatabase( const fullContent = `# ${rowTitle}${metadataSection}\n\n${content}`; await upsertNotionDocument(db, provider, row.id, `${dbTitle} — ${rowTitle}`, fullContent); - } + }); log.debug({ id: item.id, title: dbTitle, rows: rows.length }, "Indexed Notion database"); } @@ -490,12 +499,13 @@ async function syncNotionItem( item: NotionSearchResult, excludeSet: Set<string>, result: NotionSyncResult, + signal: AbortSignal | undefined, ): Promise<void> { if (item.object === "page") { const indexed = await syncNotionPage(db, provider, token, item); if (indexed) result.pagesIndexed++; } else if (item.object === "database") { - await syncNotionDatabase(db, provider, token, item, excludeSet); + await syncNotionDatabase(db, provider, token, item, excludeSet, signal); result.databasesIndexed++; } } @@ -505,6 +515,27 @@ export async function syncNotion( db: Database.Database, provider: EmbeddingProvider, config: NotionConfig, + options: ConnectorSyncOptions = {}, +): Promise<NotionSyncResult> { + return trackSync( + db, + "notion", + options.syncName ?? "notion", + () => runNotionSync(db, provider, config, options.signal), + (result) => ({ + added: result.pagesIndexed + result.databasesIndexed, + updated: 0, + deleted: 0, + errored: result.errors.length, + }), + ); +} + +async function runNotionSync( + db: Database.Database, + provider: EmbeddingProvider, + config: NotionConfig, + signal: AbortSignal | undefined, ): Promise<NotionSyncResult> { const log = getLogger(); @@ -512,58 +543,46 @@ export async function syncNotion( throw new ValidationError("Notion token must start with 'secret_' or 'ntn_'"); } - const syncId = startSync(db, "notion", "notion"); - - try { - const result: NotionSyncResult = { - pagesIndexed: 0, - databasesIndexed: 0, - errors: [], - }; - - const excludeSet = new Set(config.excludePages ?? []); + const result: NotionSyncResult = { + pagesIndexed: 0, + databasesIndexed: 0, + errors: [], + }; - log.info({ lastSync: config.lastSync }, "Starting Notion sync"); - const searchResults = await searchNotion(config.token, config.lastSync); - log.info({ count: searchResults.length }, "Found Notion objects"); + const excludeSet = new Set(config.excludePages ?? []); - for (const item of searchResults) { - if (excludeSet.has(item.id)) { - log.debug({ id: item.id }, "Skipping excluded page"); - continue; - } + log.info({ lastSync: config.lastSync }, "Starting Notion sync"); + const searchResults = await searchNotion(config.token, config.lastSync, signal); + log.info({ count: searchResults.length }, "Found Notion objects"); - try { - await syncNotionItem(db, provider, config.token, item, excludeSet, result); - } catch (err) { - const title = extractTitle(item); - const message = err instanceof Error ? err.message : String(err); - result.errors.push({ page: title, error: message }); - log.warn({ id: item.id, err }, "Failed to index Notion item"); - } + await forEachSequential(searchResults, async (item) => { + signal?.throwIfAborted(); + if (excludeSet.has(item.id)) { + log.debug({ id: item.id }, "Skipping excluded page"); + return; } - log.info( - { - pagesIndexed: result.pagesIndexed, - databasesIndexed: result.databasesIndexed, - errors: result.errors.length, - }, - "Notion sync complete", - ); + try { + await syncNotionItem(db, provider, config.token, item, excludeSet, result, signal); + } catch (err) { + if (signal?.aborted) throw err; + const title = extractTitle(item); + const message = err instanceof Error ? err.message : String(err); + result.errors.push({ page: title, error: message }); + log.warn({ id: item.id, err }, "Failed to index Notion item"); + } + }); - completeSync(db, syncId, { - added: result.pagesIndexed + result.databasesIndexed, - updated: 0, - deleted: 0, - errored: result.errors.length, - }); + log.info( + { + pagesIndexed: result.pagesIndexed, + databasesIndexed: result.databasesIndexed, + errors: result.errors.length, + }, + "Notion sync complete", + ); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; - } + return result; } /** Remove all Notion-sourced documents from the knowledge base. */ diff --git a/src/connectors/obsidian-wikilinks.ts b/src/connectors/obsidian-wikilinks.ts new file mode 100644 index 00000000..6ed076c3 --- /dev/null +++ b/src/connectors/obsidian-wikilinks.ts @@ -0,0 +1,107 @@ +/** + * Obsidian `[[wikilink]]` scanner. + * + * Finds the same links as /(?<!!)\[\[([^\]|]+)(?:\|([^\]]*))?\]\]/g: "[[", a link of one or + * more characters other than "]" and "|", an optional "|display" without "]", then "]]". + * A "[[" right after "!" is an embed, not a wikilink. The scan is linear in the text length; + * the regular expression backtracks polynomially on input such as "[[[[[[...". + */ + +export interface Wikilink { + /** Index of the opening "[[". */ + start: number; + /** Index after the closing "]]". */ + end: number; + link: string; + /** Text after "|", or undefined when there is no "|". */ + display: string | undefined; +} + +/** Outcome of reading a wikilink at one "[[": the link, or where the next candidate can start. */ +type ScanResult = { link: Wikilink } | { resumeAt: number }; + +/** + * `indexOf(char, from)` that reuses its last answer while it is still the first match at or + * after `from`. Positions only grow during a scan, so the text is searched about once. + */ +function indexFinder(text: string, char: string): (from: number) => number { + let searchedFrom = Number.POSITIVE_INFINITY; + let found = -1; + return (from) => { + const reusable = from >= searchedFrom && (found === -1 || found >= from); + if (!reusable) { + found = text.indexOf(char, from); + searchedFrom = from; + } + return found; + }; +} + +interface Scanner { + text: string; + nextClose: (from: number) => number; + nextPipe: (from: number) => number; +} + +/** `index`, or the text length when `index` is -1 (not found). */ +function orEnd(index: number, text: string): number { + return index === -1 ? text.length : index; +} + +/** + * Read the wikilink whose "[[" is at `start`. Each "[[" between `start` and the index where a + * failed scan stopped would fail the same way, so the search resumes there. + */ +function scanAt(scanner: Scanner, start: number): ScanResult { + const { text } = scanner; + if (start > 0 && text.charAt(start - 1) === "!") return { resumeAt: start + 1 }; + const linkStart = start + 2; + const pipe = scanner.nextPipe(linkStart); + const linkEnd = Math.min(orEnd(scanner.nextClose(linkStart), text), orEnd(pipe, text)); + if (linkEnd === linkStart) return { resumeAt: start + 1 }; + const link = text.slice(linkStart, linkEnd); + if (linkEnd !== pipe) { + return text.startsWith("]]", linkEnd) + ? { link: { start, end: linkEnd + 2, link, display: undefined } } + : { resumeAt: linkEnd }; + } + const displayEnd = orEnd(scanner.nextClose(pipe + 1), text); + if (!text.startsWith("]]", displayEnd)) return { resumeAt: displayEnd }; + const display = text.slice(pipe + 1, displayEnd); + return { link: { start, end: displayEnd + 2, link, display } }; +} + +/** Find every wikilink in `text`, left to right, without overlaps. */ +export function findWikilinks(text: string): Wikilink[] { + const links: Wikilink[] = []; + const scanner: Scanner = { + text, + nextClose: indexFinder(text, "]"), + nextPipe: indexFinder(text, "|"), + }; + + let start = text.indexOf("[["); + while (start !== -1) { + const scanned = scanAt(scanner, start); + let resumeAt: number; + if ("link" in scanned) { + links.push(scanned.link); + resumeAt = scanned.link.end; + } else { + resumeAt = scanned.resumeAt; + } + start = text.indexOf("[[", resumeAt); + } + return links; +} + +/** Replace each wikilink in `text` with `replace(link)`. */ +export function replaceWikilinks(text: string, replace: (link: Wikilink) => string): string { + let out = ""; + let last = 0; + for (const link of findWikilinks(text)) { + out += text.slice(last, link.start) + replace(link); + last = link.end; + } + return out + text.slice(last); +} diff --git a/src/connectors/obsidian.ts b/src/connectors/obsidian.ts index 3c59a6ad..7a06d118 100644 --- a/src/connectors/obsidian.ts +++ b/src/connectors/obsidian.ts @@ -5,13 +5,15 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; -import { createTopic, listTopics } from "../core/topics.js"; +import { createTopic } from "../core/topics.js"; import { addTagsToDocument, createTag } from "../core/tags.js"; import { createLink, resolveDocumentByTitle } from "../core/links.js"; import { getLogger } from "../logger.js"; import { ValidationError } from "../errors.js"; +import { forEachSequential } from "../utils/async.js"; import { loadConnectorConfig, saveConnectorConfig } from "./index.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { findWikilinks, replaceWikilinks } from "./obsidian-wikilinks.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; export interface ObsidianConfig { vaultPath: string; @@ -131,14 +133,7 @@ function buildVaultFileMap(vaultFiles: string[]): Map<string, string> { /** Collect all [[wikilinks]] from the body text. */ function collectWikilinks(body: string): string[] { - const wikilinks: string[] = []; - const wikilinkRegex = /(?<!!)\[\[([^\]|]+)(?:\|([^\]]*))?\]\]/g; - let wlMatch; - while ((wlMatch = wikilinkRegex.exec(body)) !== null) { - const link = wlMatch[1] ?? ""; - wikilinks.push(link); - } - return wikilinks; + return findWikilinks(body).map((w) => w.link); } /** Apply Obsidian-specific markdown transformations: embeds, wikilinks, comments, callouts. */ @@ -152,14 +147,11 @@ function transformObsidianBody(body: string, fileMap: Map<string, string>): stri }); // Resolve [[wikilinks]] - result = result.replaceAll( - /(?<!!)\[\[([^\]|]+)(?:\|([^\]]*))?\]\]/g, - (_match, link: string, display?: string) => { - const displayText = display ?? link; - const slug = link.toLowerCase().replaceAll(/\s+/g, "-"); - return `[${displayText}](${slug})`; - }, - ); + result = replaceWikilinks(result, ({ link, display }) => { + const displayText = display ?? link; + const slug = link.toLowerCase().replaceAll(/\s+/g, "-"); + return `[${displayText}](${slug})`; + }); // Strip %%comments%% result = result.replaceAll(/%%[\s\S]*?%%/g, ""); @@ -199,19 +191,24 @@ function collectTags(body: string, frontmatter: Record<string, unknown>): string return [...tagSet]; } -export function parseObsidianMarkdown( - content: string, - vaultFiles: string[], -): { +interface ParsedObsidianMarkdown { frontmatter: Record<string, unknown>; body: string; tags: string[]; wikilinks: string[]; -} { +} + +export function parseObsidianMarkdown( + content: string, + vaultFiles: string[], +): ParsedObsidianMarkdown { + return parseWithFileMap(content, buildVaultFileMap(vaultFiles)); +} + +function parseWithFileMap(content: string, fileMap: Map<string, string>): ParsedObsidianMarkdown { const safeContent = content.length > MAX_PARSE_SIZE ? content.slice(0, MAX_PARSE_SIZE) : content; const { frontmatter, body: rawBody } = extractFrontmatter(safeContent); - const fileMap = buildVaultFileMap(vaultFiles); const wikilinks = collectWikilinks(rawBody); const body = transformObsidianBody(rawBody, fileMap); const tags = collectTags(body, frontmatter); @@ -222,15 +219,9 @@ export function parseObsidianMarkdown( function resolveEmbeds( body: string, vaultPath: string, - vaultFiles: string[], + fileMap: Map<string, string>, _visited: Set<string> = new Set(), ): string { - const fileMap = new Map<string, string>(); - for (const f of vaultFiles) { - const name = basename(f, ".md"); - fileMap.set(name.toLowerCase(), f); - } - return body.replace( /!\[\[([^\]|]+)(?:\|([^\]]*))?\]\]/g, (_match, link: string, display?: string) => { @@ -261,13 +252,9 @@ function folderToTopic(relPath: string): string | undefined { return dir; } +/** createTopic returns the existing topic when the name is already taken (ON CONFLICT). */ function getOrCreateTopic(db: Database.Database, topicPath: string): string { - const existing = listTopics(db); - const found = existing.find((t) => t.name === topicPath); - if (found) return found.id; - - const topic = createTopic(db, { name: topicPath }); - return topic.id; + return createTopic(db, { name: topicPath }).id; } /** Determine the topic ID for a file based on the topic mapping strategy. */ @@ -345,7 +332,7 @@ async function processVaultFile( provider: EmbeddingProvider, config: ObsidianConfig, relPath: string, - vaultFiles: string[], + fileMap: Map<string, string>, tracked: VaultFileEntry | undefined, log: ReturnType<typeof getLogger>, ): Promise<{ entry: VaultFileEntry; isUpdate: boolean } | "unchanged"> { @@ -356,8 +343,8 @@ async function processVaultFile( if (tracked?.mtime === mtime) return "unchanged"; const rawContent = readFileSync(fullPath, "utf-8"); - const contentWithEmbeds = resolveEmbeds(rawContent, config.vaultPath, vaultFiles); - const parsed = parseObsidianMarkdown(contentWithEmbeds, vaultFiles); + const contentWithEmbeds = resolveEmbeds(rawContent, config.vaultPath, fileMap); + const parsed = parseWithFileMap(contentWithEmbeds, fileMap); const title = typeof parsed.frontmatter.title === "string" @@ -421,25 +408,31 @@ function applyVaultFileOutcome( } } +interface SyncVaultFilesContext { + db: Database.Database; + provider: EmbeddingProvider; + config: ObsidianConfig; + vaultFiles: string[]; + trackedFiles: Record<string, VaultFileEntry>; + newTrackedFiles: Record<string, VaultFileEntry>; + result: SyncResult; + signal: AbortSignal | undefined; +} + /** Sync all vault files, populating result and newTrackedFiles. */ -async function syncVaultFiles( - db: Database.Database, - provider: EmbeddingProvider, - config: ObsidianConfig, - vaultFiles: string[], - trackedFiles: Record<string, VaultFileEntry>, - newTrackedFiles: Record<string, VaultFileEntry>, - result: SyncResult, -): Promise<void> { +async function syncVaultFiles(ctx: SyncVaultFilesContext): Promise<void> { + const { db, provider, config, vaultFiles, trackedFiles, newTrackedFiles, result, signal } = ctx; const log = getLogger(); - for (const relPath of vaultFiles) { + const fileMap = buildVaultFileMap(vaultFiles); + await forEachSequential(vaultFiles, async (relPath) => { + signal?.throwIfAborted(); try { const outcome = await processVaultFile( db, provider, config, relPath, - vaultFiles, + fileMap, trackedFiles[relPath], log, ); @@ -453,13 +446,34 @@ async function syncVaultFiles( newTrackedFiles[relPath] = tracked; } } - } + }); } export async function syncObsidianVault( db: Database.Database, provider: EmbeddingProvider, config: ObsidianConfig, + options: ConnectorSyncOptions = {}, +): Promise<SyncResult> { + return trackSync( + db, + "obsidian", + options.syncName ?? config.vaultPath, + () => runObsidianSync(db, provider, config, options.signal), + (result) => ({ + added: result.added, + updated: result.updated, + deleted: result.deleted, + errored: result.errors.length, + }), + ); +} + +async function runObsidianSync( + db: Database.Database, + provider: EmbeddingProvider, + config: ObsidianConfig, + signal: AbortSignal | undefined, ): Promise<SyncResult> { const log = getLogger(); const result: SyncResult = { added: 0, updated: 0, deleted: 0, errors: [] }; @@ -468,60 +482,52 @@ export async function syncObsidianVault( throw new ValidationError("Vault path is required"); } - const syncId = startSync(db, "obsidian", config.vaultPath); + const excludePatterns = [...DEFAULT_EXCLUDE, ...config.excludePatterns]; + const vaultFiles = findMarkdownFiles(config.vaultPath, excludePatterns); - try { - const excludePatterns = [...DEFAULT_EXCLUDE, ...config.excludePatterns]; - const vaultFiles = findMarkdownFiles(config.vaultPath, excludePatterns); - - log.info( - { vaultPath: config.vaultPath, fileCount: vaultFiles.length }, - "Syncing Obsidian vault", - ); - - const connectorConfig = loadConnectorConfig(); - const vaultKey = `obsidian:${config.vaultPath}`; - const existingState = connectorConfig[vaultKey] as VaultState | undefined; - const trackedFiles = existingState?.files ?? {}; - const newTrackedFiles: Record<string, VaultFileEntry> = {}; - const currentFileSet = new Set(vaultFiles); - - await syncVaultFiles(db, provider, config, vaultFiles, trackedFiles, newTrackedFiles, result); - - result.deleted = deleteRemovedFiles(db, trackedFiles, currentFileSet); - - connectorConfig[vaultKey] = { - type: "obsidian", - vaultPath: config.vaultPath, - lastSync: new Date().toISOString(), - topicMapping: config.topicMapping, - excludePatterns: config.excludePatterns, - files: newTrackedFiles, - } satisfies VaultState; - saveConnectorConfig(connectorConfig); - - log.info( - { - added: result.added, - updated: result.updated, - deleted: result.deleted, - errors: result.errors.length, - }, - "Obsidian vault sync complete", - ); - - completeSync(db, syncId, { + log.info({ vaultPath: config.vaultPath, fileCount: vaultFiles.length }, "Syncing Obsidian vault"); + + const connectorConfig = loadConnectorConfig(); + const vaultKey = `obsidian:${config.vaultPath}`; + const existingState = connectorConfig[vaultKey] as VaultState | undefined; + const trackedFiles = existingState?.files ?? {}; + const newTrackedFiles: Record<string, VaultFileEntry> = {}; + const currentFileSet = new Set(vaultFiles); + + await syncVaultFiles({ + db, + provider, + config, + vaultFiles, + trackedFiles, + newTrackedFiles, + result, + signal, + }); + + result.deleted = deleteRemovedFiles(db, trackedFiles, currentFileSet); + + connectorConfig[vaultKey] = { + type: "obsidian", + vaultPath: config.vaultPath, + lastSync: new Date().toISOString(), + topicMapping: config.topicMapping, + excludePatterns: config.excludePatterns, + files: newTrackedFiles, + } satisfies VaultState; + saveConnectorConfig(connectorConfig); + + log.info( + { added: result.added, updated: result.updated, deleted: result.deleted, - errored: result.errors.length, - }); + errors: result.errors.length, + }, + "Obsidian vault sync complete", + ); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; - } + return result; } export function disconnectVault(db: Database.Database, vaultPath: string): number { diff --git a/src/connectors/onenote.ts b/src/connectors/onenote.ts index fb3b9474..5b756f83 100644 --- a/src/connectors/onenote.ts +++ b/src/connectors/onenote.ts @@ -1,13 +1,15 @@ import type Database from "better-sqlite3"; +import { setTimeout as sleep } from "node:timers/promises"; import { NodeHtmlMarkdown } from "node-html-markdown"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { getLogger } from "../logger.js"; import { LibScopeError } from "../errors.js"; +import { forEachSequential } from "../utils/async.js"; import { indexDocument } from "../core/indexing.js"; import { deleteDocument } from "../core/documents.js"; -import { createTopic, listTopics } from "../core/topics.js"; +import { createTopic } from "../core/topics.js"; import { loadConnectorConfig, saveConnectorConfig } from "./index.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; // --------------------------------------------------------------------------- // Types @@ -76,7 +78,7 @@ async function rateLimitedFetch(url: string, options: RequestInit): Promise<Resp if (requestTimestamps.length >= MAX_REQUESTS_PER_MINUTE) { const waitMs = 60_000 - (now - (requestTimestamps[0] ?? now)); log.debug({ waitMs }, "Rate limit reached, waiting"); - await new Promise((resolve) => setTimeout(resolve, waitMs)); + await sleep(waitMs); } unlock!(); @@ -97,7 +99,7 @@ async function rateLimitedFetch(url: string, options: RequestInit): Promise<Resp `Rate limited by Graph API (attempt ${attempt + 1})`, "ONENOTE_RATE_LIMITED", ); - await new Promise((resolve) => setTimeout(resolve, delayMs)); + await sleep(delayMs); continue; } @@ -288,46 +290,60 @@ export async function authenticateDeviceCode( const interval = (dcData.interval ?? 5) * 1000; const deadline = Date.now() + dcData.expires_in * 1000; - while (Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, interval)); - - const tokenRes = await fetch(tokenUrl, { - method: "POST", - headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: new URLSearchParams({ - grant_type: "urn:ietf:params:oauth:grant-type:device_code", - client_id: clientId, - device_code: dcData.device_code, - }), - signal: AbortSignal.timeout(30_000), - }); + return pollDeviceToken({ + tokenUrl, + clientId, + deviceCode: dcData.device_code, + interval, + deadline, + }); +} - if (tokenRes.ok) { - const tokenData = (await tokenRes.json()) as { - access_token: string; - refresh_token: string; - expires_in: number; - }; - const expiresAt = new Date(Date.now() + tokenData.expires_in * 1000).toISOString(); - return { - accessToken: tokenData.access_token, - refreshToken: tokenData.refresh_token, - expiresAt, - }; - } +/** Ask for the token every `interval` ms until the user signs in, an error, or `deadline`. */ +async function pollDeviceToken(poll: { + tokenUrl: string; + clientId: string; + deviceCode: string; + interval: number; + deadline: number; +}): Promise<{ accessToken: string; refreshToken: string; expiresAt: string }> { + if (Date.now() >= poll.deadline) { + throw new LibScopeError("Device code authentication timed out", "ONENOTE_AUTH_ERROR"); + } + await sleep(poll.interval); - const errData = (await tokenRes.json()) as { error: string }; - if (errData.error === "authorization_pending") { - continue; - } - if (errData.error === "slow_down") { - await new Promise((resolve) => setTimeout(resolve, 5000)); - continue; - } - throw new LibScopeError(`Authentication failed: ${errData.error}`, "ONENOTE_AUTH_ERROR"); + const tokenRes = await fetch(poll.tokenUrl, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "urn:ietf:params:oauth:grant-type:device_code", + client_id: poll.clientId, + device_code: poll.deviceCode, + }), + signal: AbortSignal.timeout(30_000), + }); + + if (tokenRes.ok) { + const tokenData = (await tokenRes.json()) as { + access_token: string; + refresh_token: string; + expires_in: number; + }; + const expiresAt = new Date(Date.now() + tokenData.expires_in * 1000).toISOString(); + return { + accessToken: tokenData.access_token, + refreshToken: tokenData.refresh_token, + expiresAt, + }; } - throw new LibScopeError("Device code authentication timed out", "ONENOTE_AUTH_ERROR"); + const errData = (await tokenRes.json()) as { error: string }; + if (errData.error === "slow_down") { + await sleep(5000); + } else if (errData.error !== "authorization_pending") { + throw new LibScopeError(`Authentication failed: ${errData.error}`, "ONENOTE_AUTH_ERROR"); + } + return pollDeviceToken(poll); } // --------------------------------------------------------------------------- @@ -380,13 +396,9 @@ export async function refreshAccessToken( // Sync // --------------------------------------------------------------------------- +/** createTopic returns the existing topic when the name is already taken (ON CONFLICT). */ function ensureOrCreateTopic(db: Database.Database, name: string, parentId?: string): string { - const existing = listTopics(db, parentId).find((t) => t.name === name); - if (existing) { - return existing.id; - } - const topic = createTopic(db, { name, parentId }); - return topic.id; + return createTopic(db, { name, parentId }).id; } function buildSourceUrl(notebook: string, section: string, pageTitle: string): string { @@ -444,6 +456,7 @@ interface SyncOneNoteSectionOptions { config: OneNoteConfig; seenSourceUrls: Set<string>; result: OneNoteSyncResult; + signal: AbortSignal | undefined; } /** Sync all pages within a single section. */ @@ -458,6 +471,7 @@ async function syncOneNoteSection(options: SyncOneNoteSectionOptions): Promise<v config, seenSourceUrls, result, + signal, } = options; const log = getLogger(); const sectionTopicId = ensureOrCreateTopic(db, section.displayName, notebookTopicId); @@ -474,7 +488,8 @@ async function syncOneNoteSection(options: SyncOneNoteSectionOptions): Promise<v return; } - for (const page of pages) { + await forEachSequential(pages, async (page) => { + signal?.throwIfAborted(); const sourceUrl = buildSourceUrl(notebookName, section.displayName, page.title); seenSourceUrls.add(sourceUrl); @@ -495,19 +510,16 @@ async function syncOneNoteSection(options: SyncOneNoteSectionOptions): Promise<v log.error({ page: page.title, err }, "Failed to sync page"); result.errors.push({ page: page.title, error: msg }); } - } + }); } /** Sync all sections within a single notebook. */ async function syncOneNoteNotebook( - db: Database.Database, - provider: EmbeddingProvider, - token: string, - notebook: GraphNotebook, - config: OneNoteConfig, - seenSourceUrls: Set<string>, - result: OneNoteSyncResult, + options: Omit<SyncOneNoteSectionOptions, "notebookName" | "section" | "notebookTopicId"> & { + notebook: GraphNotebook; + }, ): Promise<void> { + const { db, provider, token, notebook, config, seenSourceUrls, result, signal } = options; const notebookTopicId = ensureOrCreateTopic(db, notebook.displayName); let sections: GraphSection[]; @@ -522,7 +534,8 @@ async function syncOneNoteNotebook( const filteredSections = sections.filter((s) => !config.excludeSections.includes(s.displayName)); result.sections += filteredSections.length; - for (const section of filteredSections) { + await forEachSequential(filteredSections, async (section) => { + signal?.throwIfAborted(); await syncOneNoteSection({ db, provider, @@ -533,8 +546,9 @@ async function syncOneNoteNotebook( config, seenSourceUrls, result, + signal, }); - } + }); } /** Delete OneNote documents whose source URLs were not seen during sync. */ @@ -552,76 +566,120 @@ function deleteStaleOneNoteDocs(db: Database.Database, seenSourceUrls: Set<strin return deleted; } +const TOKEN_REFRESH_MARGIN_MS = 5 * 60_000; + +/** True when the access token is missing, has no known expiry, or expires soon. */ +function accessTokenNeedsRefresh(config: OneNoteConfig): boolean { + if (!config.accessToken || !config.tokenExpiry) return true; + const expiresAt = Date.parse(config.tokenExpiry); + return !Number.isNaN(expiresAt) && expiresAt - Date.now() < TOKEN_REFRESH_MARGIN_MS; +} + +/** + * Refresh `config.accessToken` in place when it is missing, has no recorded expiry, or is + * about to expire, and the config holds a client ID and refresh token. Updates accessToken, + * refreshToken and tokenExpiry on the config object; callers that persist the config should + * save it after the sync. Returns true when the tokens were refreshed. + */ +export async function ensureOneNoteAccessToken(config: OneNoteConfig): Promise<boolean> { + if (!config.clientId || !config.refreshToken || !accessTokenNeedsRefresh(config)) { + return false; + } + const tenantId = config.tenantId === "" ? undefined : config.tenantId; + const auth = await refreshAccessToken(config.clientId, config.refreshToken, tenantId); + config.accessToken = auth.accessToken; + config.refreshToken = auth.refreshToken; + config.tokenExpiry = auth.expiresAt; + getLogger().info("Refreshed OneNote access token"); + return true; +} + +/** + * Sync OneNote notebooks. Refreshes an expired or missing access token first when the + * config carries a refresh token (see ensureOneNoteAccessToken; the config is updated). + */ export async function syncOneNote( db: Database.Database, provider: EmbeddingProvider, config: OneNoteConfig, + options: ConnectorSyncOptions = {}, +): Promise<OneNoteSyncResult> { + return trackSync( + db, + "onenote", + options.syncName ?? "onenote", + () => runOneNoteSync(db, provider, config, options.signal), + (result) => ({ + added: result.pagesAdded, + updated: result.pagesUpdated, + deleted: result.pagesDeleted, + errored: result.errors.length, + }), + ); +} + +async function runOneNoteSync( + db: Database.Database, + provider: EmbeddingProvider, + config: OneNoteConfig, + signal: AbortSignal | undefined, ): Promise<OneNoteSyncResult> { const log = getLogger(); + await ensureOneNoteAccessToken(config); const token = config.accessToken; if (!token) { throw new LibScopeError("No access token provided", "ONENOTE_AUTH_ERROR"); } - const syncId = startSync(db, "onenote", "onenote"); - - try { - const result: OneNoteSyncResult = { - notebooks: 0, - sections: 0, - pagesAdded: 0, - pagesUpdated: 0, - pagesDeleted: 0, - errors: [], - }; - - log.info("Starting OneNote sync"); - - const allNotebooks = await listNotebooks(token); - const targetNotebooks = - config.notebooks.length === 1 && config.notebooks[0] === "all" - ? allNotebooks - : allNotebooks.filter((nb) => config.notebooks.includes(nb.displayName)); + const result: OneNoteSyncResult = { + notebooks: 0, + sections: 0, + pagesAdded: 0, + pagesUpdated: 0, + pagesDeleted: 0, + errors: [], + }; - result.notebooks = targetNotebooks.length; - const seenSourceUrls = new Set<string>(); + log.info("Starting OneNote sync"); - for (const notebook of targetNotebooks) { - await syncOneNoteNotebook(db, provider, token, notebook, config, seenSourceUrls, result); - } + const allNotebooks = await listNotebooks(token); + const targetNotebooks = + config.notebooks.length === 1 && config.notebooks[0] === "all" + ? allNotebooks + : allNotebooks.filter((nb) => config.notebooks.includes(nb.displayName)); - result.pagesDeleted = deleteStaleOneNoteDocs(db, seenSourceUrls); - - const connConfig = loadConnectorConfig(); - const onenoteConf = (connConfig.onenote ?? {}) as Record<string, unknown>; - onenoteConf.lastSync = new Date().toISOString(); - connConfig.onenote = onenoteConf; - saveConnectorConfig(connConfig); - - log.info( - { - notebooks: result.notebooks, - sections: result.sections, - pagesAdded: result.pagesAdded, - pagesUpdated: result.pagesUpdated, - pagesDeleted: result.pagesDeleted, - errors: result.errors.length, - }, - "OneNote sync complete", - ); + result.notebooks = targetNotebooks.length; + const seenSourceUrls = new Set<string>(); - completeSync(db, syncId, { - added: result.pagesAdded, - updated: result.pagesUpdated, - deleted: result.pagesDeleted, - errored: result.errors.length, + await forEachSequential(targetNotebooks, async (notebook) => { + signal?.throwIfAborted(); + await syncOneNoteNotebook({ + db, + provider, + token, + notebook, + config, + seenSourceUrls, + result, + signal, }); + }); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; - } + result.pagesDeleted = deleteStaleOneNoteDocs(db, seenSourceUrls); + + log.info( + { + notebooks: result.notebooks, + sections: result.sections, + pagesAdded: result.pagesAdded, + pagesUpdated: result.pagesUpdated, + pagesDeleted: result.pagesDeleted, + errors: result.errors.length, + }, + "OneNote sync complete", + ); + + return result; } // --------------------------------------------------------------------------- diff --git a/src/connectors/registry.ts b/src/connectors/registry.ts new file mode 100644 index 00000000..d59f45bb --- /dev/null +++ b/src/connectors/registry.ts @@ -0,0 +1,404 @@ +/** + * Connector registry: one table describing every connector type (its config shape, which + * fields are secrets, how to sync it and how to remove its data). `libscope sync`, + * `libscope disconnect`, the MCP/REST sync operations and the scheduler all go through it. + * Connector settings are the saved configs in ~/.libscope/connectors/<name>.json. + */ +import type Database from "better-sqlite3"; +import cron from "node-cron"; +import { z } from "zod"; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import { NotFoundError, ValidationError } from "../errors.js"; +import { + deleteNamedConnectorConfig, + hasNamedConnectorConfig, + listNamedConnectorConfigs, + loadNamedConnectorConfig, + saveNamedConnectorConfig, +} from "./index.js"; +import { + isConnectorType, + loadSavedConnectorConfig, + resolveConnectorType, + syncSavedConnector, + type ConnectorType, +} from "./saved-config.js"; +import { getConnectorStatus, type ConnectorSyncOptions } from "./sync-tracker.js"; +import { + syncNotion, + disconnectNotion, + type NotionConfig, + type NotionSyncResult, +} from "./notion.js"; +import { syncSlack, disconnectSlack, type SlackConfig, type SlackSyncResult } from "./slack.js"; +import { + syncConfluence, + disconnectConfluence, + type ConfluenceConfig, + type ConfluenceSyncResult, +} from "./confluence.js"; +import { + syncObsidianVault, + disconnectVault, + type ObsidianConfig, + type SyncResult as ObsidianSyncResult, +} from "./obsidian.js"; +import { + syncOneNote, + disconnectOneNote, + type OneNoteConfig, + type OneNoteSyncResult, +} from "./onenote.js"; +import { + syncDocSite, + disconnectDocSite, + type DocSiteConfig, + type DocSiteSyncResult, +} from "./docs.js"; + +/** Counts from one sync, the same for every connector type. */ +export interface ConnectorSyncSummary { + added: number; + updated: number; + deleted: number; + /** One line per item that failed. */ + errors: string[]; +} + +export interface ConnectorDefinition<C = Record<string, unknown>, R = unknown> { + type: ConnectorType; + /** Human-readable name, e.g. "Confluence". */ + label: string; + /** Shape of the saved config (the connector's own settings). */ + configSchema: z.ZodObject; + /** Config fields that hold credentials; never shown in listings. */ + secretFields: readonly string[]; + sync( + db: Database.Database, + provider: EmbeddingProvider, + config: C, + options: ConnectorSyncOptions, + ): Promise<R>; + summarize(result: R): ConnectorSyncSummary; + /** Remove the documents this connector created. `config` is the saved config, if any. */ + disconnect(db: Database.Database, config: C | undefined): number | Promise<number>; +} + +/** Erase the config/result types so definitions fit in one table. */ +function defineConnector<C, R>(def: ConnectorDefinition<C, R>): ConnectorDefinition { + return def as unknown as ConnectorDefinition; +} + +function requireField<C>(config: C | undefined, field: keyof C & string, type: string): C { + if (!config?.[field]) { + throw new ValidationError(`Disconnecting ${type} needs the saved connection's ${field}`); + } + return config; +} + +const errorLines = <E>(errors: E[], format: (e: E) => string): string[] => errors.map(format); + +// Each sync/disconnect is wrapped in an arrow so connector modules are only touched when used. +export const CONNECTORS: Readonly<Record<ConnectorType, ConnectorDefinition>> = { + notion: defineConnector<NotionConfig, NotionSyncResult>({ + type: "notion", + label: "Notion", + configSchema: z.object({ + token: z.string().describe("Notion integration token (secret_... or ntn_...)"), + excludePages: z.array(z.string()).optional().describe("Page/database IDs to skip"), + }), + secretFields: ["token"], + sync: (db, provider, config, options) => syncNotion(db, provider, config, options), + summarize: (r) => ({ + added: r.pagesIndexed + r.databasesIndexed, + updated: 0, + deleted: 0, + errors: errorLines(r.errors, (e) => `${e.page}: ${e.error}`), + }), + disconnect: (db) => disconnectNotion(db), + }), + slack: defineConnector<SlackConfig, SlackSyncResult>({ + type: "slack", + label: "Slack", + configSchema: z.object({ + token: z.string().describe("Slack bot or user token (xoxb-... or xoxp-...)"), + channels: z.array(z.string()).describe("Channel names or IDs, or ['all']"), + excludeChannels: z.array(z.string()).optional().describe("Channel names to skip"), + threadMode: z.enum(["aggregate", "separate"]).describe("One document per thread or reply"), + }), + secretFields: ["token"], + sync: (db, provider, config, options) => syncSlack(db, provider, config, options), + summarize: (r) => ({ + added: r.messagesIndexed + r.threadsIndexed, + updated: 0, + deleted: 0, + errors: errorLines(r.errors, (e) => `#${e.channel}: ${e.error}`), + }), + disconnect: (db) => disconnectSlack(db), + }), + confluence: defineConnector<ConfluenceConfig, ConfluenceSyncResult>({ + type: "confluence", + label: "Confluence", + configSchema: z.object({ + baseUrl: z.string().describe("Confluence base URL, e.g. https://acme.atlassian.net"), + type: z.enum(["cloud", "server"]).optional().describe("Cloud or Server/Data Center"), + email: z.string().optional().describe("User email (Cloud)"), + token: z.string().describe("API token (Cloud) or personal access token (Server)"), + spaces: z.array(z.string()).describe("Space keys, or ['all']"), + excludeSpaces: z.array(z.string()).optional().describe("Space keys to skip"), + }), + secretFields: ["token"], + sync: (db, provider, config, options) => syncConfluence(db, provider, config, options), + summarize: (r) => ({ + added: r.pagesIndexed, + updated: r.pagesUpdated, + deleted: 0, + errors: errorLines(r.errors, (e) => `${e.page}: ${e.error}`), + }), + disconnect: (db) => disconnectConfluence(db), + }), + obsidian: defineConnector<ObsidianConfig, ObsidianSyncResult>({ + type: "obsidian", + label: "Obsidian", + configSchema: z.object({ + vaultPath: z.string().describe("Absolute path of the vault"), + topicMapping: z.enum(["folder", "frontmatter"]).describe("Where topics come from"), + excludePatterns: z.array(z.string()).describe("Globs for files to skip"), + }), + secretFields: [], + sync: (db, provider, config, options) => syncObsidianVault(db, provider, config, options), + summarize: (r) => ({ + added: r.added, + updated: r.updated, + deleted: r.deleted, + errors: errorLines(r.errors, (e) => `${e.file}: ${e.error}`), + }), + disconnect: (db, config) => + disconnectVault(db, requireField(config, "vaultPath", "obsidian").vaultPath), + }), + onenote: defineConnector<OneNoteConfig, OneNoteSyncResult>({ + type: "onenote", + label: "OneNote", + configSchema: z.object({ + clientId: z.string().describe("Azure app client ID"), + tenantId: z.string().describe("Azure tenant ID (default: common)"), + accessToken: z.string().optional().describe("Microsoft Graph access token"), + refreshToken: z.string().optional().describe("Refresh token from device sign-in"), + notebooks: z.array(z.string()).describe("Notebook names, or ['all']"), + excludeSections: z.array(z.string()).describe("Section names to skip"), + }), + secretFields: ["accessToken", "refreshToken"], + sync: (db, provider, config, options) => syncOneNote(db, provider, config, options), + summarize: (r) => ({ + added: r.pagesAdded, + updated: r.pagesUpdated, + deleted: r.pagesDeleted, + errors: errorLines(r.errors, (e) => `${e.page}: ${e.error}`), + }), + disconnect: (db) => disconnectOneNote(db), + }), + docs: defineConnector<DocSiteConfig, DocSiteSyncResult>({ + type: "docs", + label: "Documentation site", + configSchema: z.object({ + url: z.string().describe("Root URL of the documentation site"), + type: z + .enum(["auto", "sphinx", "vitepress", "doxygen", "generic"]) + .optional() + .describe("Site generator (default: auto-detect)"), + library: z.string().optional().describe("Library name for indexed pages"), + version: z.string().optional().describe("Library version for indexed pages"), + maxPages: z.number().int().positive().optional().describe("Page limit (default 500)"), + maxDepth: z.number().int().min(0).optional().describe("Link depth limit (default 10)"), + pathPrefix: z.string().optional().describe("Only crawl URLs under this path"), + }), + secretFields: [], + sync: (db, provider, config, options) => syncDocSite(db, provider, config, options), + summarize: (r) => ({ + added: r.pagesIndexed, + updated: r.pagesUpdated, + deleted: 0, + errors: errorLines(r.errors, (e) => `${e.url}: ${e.error}`), + }), + disconnect: (db, config) => disconnectDocSite(db, requireField(config, "url", "docs").url), + }), +}; + +/** The definition for `type`. Throws ValidationError for an unknown type. */ +export function getConnector(type: string): ConnectorDefinition { + if (!isConnectorType(type)) { + throw new ValidationError( + `Unknown connector type "${type}". Known types: ${Object.keys(CONNECTORS).join(", ")}`, + ); + } + return CONNECTORS[type]; +} + +/** + * Re-sync saved connector `name` of type `type` with its saved config (used by the scheduler + * and the sync operation). Writes exactly one connector_syncs row, recorded under `name`, and + * saves the new lastSync. Errors are rethrown. + */ +export async function runSavedConnectorSync( + db: Database.Database, + provider: EmbeddingProvider, + type: ConnectorType, + name: string, + options: ConnectorSyncOptions = {}, +): Promise<ConnectorSyncSummary> { + const connector = CONNECTORS[type]; + const result = await syncSavedConnector<{ lastSync?: string | undefined }, unknown>( + db, + type, + name, + (config) => connector.sync(db, provider, config, { ...options, syncName: name }), + ); + return connector.summarize(result); +} + +/** Connector type of saved connection `name`. Throws NotFoundError when it does not exist. */ +export function savedConnectionType(name: string): ConnectorType { + if (!hasNamedConnectorConfig(name)) { + throw new NotFoundError( + `No saved connection named "${name}". Create one with 'libscope connect <type>'.`, + "CONNECTION_NOT_FOUND", + ); + } + const type = resolveConnectorType(name, loadNamedConnectorConfig<Record<string, unknown>>(name)); + return getConnector(type).type; +} + +/** Sync saved connection `name`. */ +export async function syncConnection( + db: Database.Database, + provider: EmbeddingProvider, + name: string, + options: ConnectorSyncOptions = {}, +): Promise<{ name: string; type: ConnectorType; summary: ConnectorSyncSummary }> { + const type = savedConnectionType(name); + const summary = await runSavedConnectorSync(db, provider, type, name, options); + return { name, type, summary }; +} + +export interface ConnectionInfo { + name: string; + type: string; + /** Saved settings with secret fields replaced by "***". */ + settings: Record<string, unknown>; + schedule?: string | undefined; + lastSync?: string | undefined; + /** Status of the most recent recorded run, if any. */ + lastRun?: { status: string; startedAt: string; error: string | null } | undefined; +} + +/** + * Create or update saved connection `name` of type `type`: `settings` (undefined values + * ignored) are merged over the connection's saved settings and checked against the type's + * config schema. `schedule`: a cron expression sets it, null removes it, undefined keeps it. + * Returns the merged settings (including secrets). + */ +export function saveConnection( + name: string, + type: string, + settings: Record<string, unknown>, + options: { schedule?: string | null | undefined } = {}, +): Record<string, unknown> { + const connector = getConnector(type); + const saved = hasNamedConnectorConfig(name) + ? loadSavedConnectorConfig<Record<string, unknown>>(connector.type, name) + : {}; + const merged: Record<string, unknown> = { ...saved }; + for (const [key, value] of Object.entries(settings)) { + if (value !== undefined) merged[key] = value; + } + const parsed = connector.configSchema.safeParse(merged); + if (!parsed.success) { + const fields = [...new Set(parsed.error.issues.map((issue) => issue.path.join(".")))]; + throw new ValidationError(`Invalid ${connector.label} settings: ${fields.join(", ")}`); + } + if (options.schedule === null) { + delete merged["schedule"]; + } else if (options.schedule !== undefined) { + if (!cron.validate(options.schedule)) { + throw new ValidationError( + `Invalid cron expression "${options.schedule}" (example: "0 */6 * * *" = every 6 hours)`, + ); + } + merged["schedule"] = { cronExpression: options.schedule }; + } + saveNamedConnectorConfig(name, { ...merged, connectorType: connector.type }); + return merged; +} + +/** Hide secret values of a saved config (the field stays so users see it is set). */ +export function maskConnectorSecrets( + type: string, + config: Record<string, unknown>, +): Record<string, unknown> { + const secrets = new Set(isConnectorType(type) ? CONNECTORS[type].secretFields : []); + return Object.fromEntries( + Object.entries(config) + .filter(([key]) => key !== "schedule" && key !== "lastSync" && key !== "connectorType") + .map(([key, value]) => [key, secrets.has(key) && value ? "***" : value]), + ); +} + +/** Every saved connection with masked settings, schedule and last run. */ +export function listConnections(db: Database.Database): ConnectionInfo[] { + return listNamedConnectorConfigs().map(({ name, config }) => { + const type = resolveConnectorType(name, config); + const schedule = config["schedule"] as { cronExpression?: string } | undefined; + const lastSync = typeof config["lastSync"] === "string" ? config["lastSync"] : undefined; + const last = getConnectorStatus(db, undefined, name)[0]; + return { + name, + type, + settings: maskConnectorSecrets(type, config), + ...(schedule?.cronExpression ? { schedule: schedule.cronExpression } : {}), + ...(lastSync ? { lastSync } : {}), + ...(last + ? { + lastRun: { + status: last.status, + startedAt: last.started_at, + error: last.error_message, + }, + } + : {}), + }; + }); +} + +/** + * Disconnect saved connection `name`: remove the documents its connector created (unless + * `keepDocuments`) and delete the saved config (it may hold credentials). Without a saved + * config, `type` (or a `name` that is a connector type) selects the connector and only + * documents are removed; otherwise an unknown name throws NotFoundError. + */ +export async function disconnectConnection( + db: Database.Database, + name: string, + options: { type?: string | undefined; keepDocuments?: boolean | undefined } = {}, +): Promise<{ + name: string; + type: ConnectorType; + documentsRemoved: number; + configRemoved: boolean; +}> { + const saved = hasNamedConnectorConfig(name); + // Without a saved connection, `name` may be a connector type (connection saved before names). + const fallbackType = options.type ?? (isConnectorType(name) ? name : undefined); + const type = + saved || fallbackType === undefined + ? savedConnectionType(name) + : getConnector(fallbackType).type; + if (options.type && options.type !== type) { + throw new ValidationError(`Connection "${name}" is a ${type} connection, not ${options.type}`); + } + const config = saved ? loadSavedConnectorConfig<Record<string, unknown>>(type, name) : undefined; + const documentsRemoved = options.keepDocuments + ? 0 + : await CONNECTORS[type].disconnect(db, config); + const configRemoved = saved && deleteNamedConnectorConfig(name); + return { name, type, documentsRemoved, configRemoved }; +} diff --git a/src/connectors/saved-config.ts b/src/connectors/saved-config.ts new file mode 100644 index 00000000..1131226a --- /dev/null +++ b/src/connectors/saved-config.ts @@ -0,0 +1,229 @@ +/** + * Saved connector configs: one file per named connector in ~/.libscope/connectors/<name>.json + * (mode 0600). `libscope connect <type>` writes them; `libscope sync`, the scheduler and the + * sync operation read them. + */ +import type Database from "better-sqlite3"; +import { ConfigError, ValidationError } from "../errors.js"; +import { + hasNamedConnectorConfig, + loadConnectorConfig, + loadNamedConnectorConfig, + saveNamedConnectorConfig, +} from "./index.js"; +import { startSync, failSync } from "./sync-tracker.js"; +import type { OneNoteConfig } from "./onenote.js"; + +/** Connector types that can be saved, re-synced and scheduled. */ +export const CONNECTOR_TYPES = [ + "notion", + "slack", + "confluence", + "obsidian", + "onenote", + "docs", +] as const; +export type ConnectorType = (typeof CONNECTOR_TYPES)[number]; + +export function isConnectorType(value: unknown): value is ConnectorType { + return typeof value === "string" && (CONNECTOR_TYPES as readonly string[]).includes(value); +} + +/** Fields stored next to the connector's own settings. */ +export interface SavedConnectorFields { + /** Connector type the config belongs to. */ + connectorType?: ConnectorType | undefined; + /** Cron schedule set by `libscope connect <type> --schedule`. */ + schedule?: { cronExpression: string } | undefined; +} + +interface HasLastSync { + lastSync?: string | undefined; +} + +/** + * Connector type a saved config declares: its `connectorType` field, else a legacy `type` + * field that names a connector (Confluence's own `type` is "cloud"/"server"). + */ +function declaredConnectorType(raw: Record<string, unknown>): ConnectorType | undefined { + if (isConnectorType(raw["connectorType"])) return raw["connectorType"]; + if (isConnectorType(raw["type"])) return raw["type"]; + return undefined; +} + +/** Connector type of a saved config: the type it declares, else the config name. */ +export function resolveConnectorType(name: string, raw: Record<string, unknown>): string { + return declaredConnectorType(raw) ?? name; +} + +/** + * Load the saved config `name` for `type`. Throws ConfigError if it is missing or declares + * a different connector type. + */ +export function loadSavedConnectorConfig<T>(type: ConnectorType, name: string = type): T { + const raw = loadNamedConnectorConfig<Record<string, unknown>>(name); + const declared = declaredConnectorType(raw); + if (declared !== undefined && declared !== type) { + throw new ConfigError(`Connector config "${name}" is a ${declared} connector, not ${type}`); + } + return raw as T; +} + +/** Like loadSavedConnectorConfig, but returns undefined when no config named `name` exists. */ +export function findSavedConnectorConfig<T>( + type: ConnectorType, + name: string = type, +): T | undefined { + return hasNamedConnectorConfig(name) ? loadSavedConnectorConfig<T>(type, name) : undefined; +} + +/** + * Saved OneNote config `name`. For the default name, falls back to the credentials that + * older versions stored under the "onenote" key of ~/.libscope/connectors.json. + */ +export function findSavedOneNoteConfig(name: string = "onenote"): OneNoteConfig | undefined { + const saved = findSavedConnectorConfig<OneNoteConfig>("onenote", name); + if (saved || name !== "onenote") return saved; + const legacy = loadConnectorConfig()["onenote"] as Partial<OneNoteConfig> | undefined; + if (!legacy?.accessToken && !legacy?.refreshToken) return undefined; + return { + clientId: legacy.clientId ?? "", + tenantId: legacy.tenantId ?? "common", + accessToken: legacy.accessToken, + refreshToken: legacy.refreshToken, + tokenExpiry: legacy.tokenExpiry, + lastSync: legacy.lastSync, + notebooks: legacy.notebooks ?? ["all"], + excludeSections: legacy.excludeSections ?? [], + }; +} + +/** + * Save connector settings as config `name`, tagged with the connector type. A schedule + * already stored under that name is kept. + */ +export function saveConnectorSettings(type: ConnectorType, name: string, settings: object): void { + const existing = hasNamedConnectorConfig(name) + ? loadNamedConnectorConfig<SavedConnectorFields>(name) + : undefined; + saveNamedConnectorConfig(name, { + ...settings, + connectorType: type, + ...(existing?.schedule ? { schedule: existing.schedule } : {}), + }); +} + +/** + * Run `sync` with `config`, then save the config as `name` with lastSync set to the time + * the sync started. Changes the connector made to the config object (refreshed OneNote + * tokens) are saved too. + */ +export async function runAndSaveConnector<C extends HasLastSync, R>( + type: ConnectorType, + name: string, + config: C, + sync: (config: C) => Promise<R>, +): Promise<R> { + const startedAt = new Date().toISOString(); + const result = await sync(config); + saveConnectorSettings(type, name, { ...config, lastSync: startedAt }); + return result; +} + +/** Record one failed connector_syncs row for a run that never reached the connector. */ +export function recordFailedSync( + db: Database.Database, + connectorType: string, + connectorName: string, + err: unknown, +): void { + failSync( + db, + startSync(db, connectorType, connectorName), + err instanceof Error ? err.message : String(err), + ); +} + +/** + * Re-sync saved connector `name` with its saved config and save the new lastSync. + * The connector records the run in connector_syncs under `name`; when the config cannot be + * loaded, one failed row is recorded here instead. Errors are rethrown. + */ +export async function syncSavedConnector<C extends HasLastSync, R>( + db: Database.Database, + type: ConnectorType, + name: string, + sync: (config: C) => Promise<R>, +): Promise<R> { + let config: C; + try { + config = loadSavedConnectorConfig<C>(type, name); + } catch (err) { + recordFailedSync(db, type, name, err); + throw err; + } + return runAndSaveConnector(type, name, config, sync); +} + +/** Per-call values for resolveSyncConfig; undefined means "not given". */ +export type SyncConfigOverrides<C> = { [K in keyof C]?: C[K] | undefined }; + +/** Settings for resolveSyncConfig. */ +export interface ResolveSyncConfigOptions<C> { + /** Values used when neither the saved config nor the call provides one. */ + defaults?: Partial<C> | undefined; + /** Fields that must be non-empty after merging. */ + required?: ReadonlyArray<keyof C & string> | undefined; +} + +/** + * Config for an on-demand sync (the MCP sync tools): `defaults`, then saved config `name` + * when it exists, then every defined value in `overrides`. Lets callers omit secrets that + * are already saved. Throws ValidationError naming required fields that are still empty. + */ +export function resolveSyncConfig<C extends object>( + type: ConnectorType, + name: string, + overrides: SyncConfigOverrides<C>, + options: ResolveSyncConfigOptions<C> = {}, +): { config: C; saved: C | undefined } { + const saved = + type === "onenote" + ? (findSavedOneNoteConfig(name) as C | undefined) + : findSavedConnectorConfig<C>(type, name); + const defined = Object.fromEntries( + Object.entries(overrides).filter(([, value]) => value !== undefined), + ) as Partial<C>; + const config: Partial<C> = { ...options.defaults, ...saved, ...defined }; + const missing = (options.required ?? []).filter((key) => { + const value = config[key]; + return value === undefined || value === ""; + }); + if (missing.length > 0) { + throw new ValidationError( + `Missing ${missing.join(", ")} for ${type} sync. Pass the value as a parameter, ` + + `or save a connector named "${name}" with 'libscope connect ${type}'.`, + ); + } + return { config: config as C, saved }; +} + +/** + * After a OneNote sync that used saved config `name`, save tokens the connector refreshed + * so a rotated refresh token is not lost. Does nothing when no refresh happened or the + * caller supplied its own access token (no refresh token in `used`). + */ +export function saveRefreshedOneNoteTokens( + name: string, + saved: OneNoteConfig | undefined, + used: OneNoteConfig, +): void { + if (!saved || !used.refreshToken) return; + if (used.accessToken === saved.accessToken && used.refreshToken === saved.refreshToken) return; + saveConnectorSettings("onenote", name, { + ...saved, + accessToken: used.accessToken, + refreshToken: used.refreshToken, + tokenExpiry: used.tokenExpiry, + }); +} diff --git a/src/connectors/slack.ts b/src/connectors/slack.ts index 26dc1cda..73648b71 100644 --- a/src/connectors/slack.ts +++ b/src/connectors/slack.ts @@ -1,10 +1,12 @@ import type Database from "better-sqlite3"; +import { setTimeout as delay } from "node:timers/promises"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { indexDocument } from "../core/indexing.js"; import { getLogger } from "../logger.js"; import { LibScopeError, ValidationError } from "../errors.js"; +import { forEachSequential, mapSequential } from "../utils/async.js"; import { fetchWithRetry } from "./http-utils.js"; -import { startSync, completeSync, failSync } from "./sync-tracker.js"; +import { trackSync, type ConnectorSyncOptions } from "./sync-tracker.js"; import { deleteDocumentRows } from "./index.js"; export interface SlackConfig { @@ -69,10 +71,6 @@ export function _clearUserCache(): void { userCache.clear(); } -async function delay(ms: number): Promise<void> { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - async function slackApi( method: string, token: string, @@ -177,38 +175,55 @@ export async function resolveUserMentions(text: string, token: string): Promise< const mentions = [...text.matchAll(mentionRegex)]; let result = text; - for (const match of mentions) { + await forEachSequential(mentions, async (match) => { const userId = match[1]; if (userId) { const displayName = await resolveUser(token, userId); result = result.replace(match[0], `@${displayName}`); } - } + }); return result; } -async function listChannels(token: string): Promise<SlackChannel[]> { - const channels: SlackChannel[] = []; - let cursor: string | undefined; +/** + * Call a paginated Slack method and collect `pick(response)` from every page, one page at a + * time with the rate-limit delay after each call. + */ +async function slackApiAllPages<T>( + method: string, + token: string, + params: Record<string, string>, + pick: (data: SlackApiResponse) => T[] | undefined, + signal: AbortSignal | undefined, +): Promise<T[]> { + const items: T[] = []; - do { - const params: Record<string, string> = { - types: "public_channel,private_channel", - limit: "200", - }; - if (cursor) params["cursor"] = cursor; + const fetchPage = async (cursor: string | undefined): Promise<void> => { + signal?.throwIfAborted(); + const pageParams = cursor ? { ...params, cursor } : params; - const data = await slackApi("conversations.list", token, params); + const data = await slackApi(method, token, pageParams); await delay(rateLimitDelayMs); - if (data.channels) { - channels.push(...data.channels); - } - cursor = data.response_metadata?.next_cursor ?? undefined; - } while (cursor); + const page = pick(data); + if (page) items.push(...page); + const next = data.response_metadata?.next_cursor ?? undefined; + if (next) await fetchPage(next); + }; - return channels; + await fetchPage(undefined); + return items; +} + +function listChannels(token: string, signal?: AbortSignal): Promise<SlackChannel[]> { + return slackApiAllPages( + "conversations.list", + token, + { types: "public_channel,private_channel", limit: "200" }, + (data) => data.channels, + signal, + ); } function filterChannels( @@ -231,32 +246,19 @@ function filterChannels( return filtered.filter((ch) => !excludeSet.has(ch.name.toLowerCase()) && !excludeSet.has(ch.id)); } -async function fetchMessages( +function fetchMessages( token: string, channelId: string, oldest?: string, + signal?: AbortSignal, ): Promise<SlackMessage[]> { - const messages: SlackMessage[] = []; - let cursor: string | undefined; - - do { - const params: Record<string, string> = { - channel: channelId, - limit: "200", - }; - if (oldest) params["oldest"] = oldest; - if (cursor) params["cursor"] = cursor; - - const data = await slackApi("conversations.history", token, params); - await delay(rateLimitDelayMs); - - if (data.messages) { - messages.push(...data.messages); - } - cursor = data.response_metadata?.next_cursor ?? undefined; - } while (cursor); + const params: Record<string, string> = { + channel: channelId, + limit: "200", + }; + if (oldest) params["oldest"] = oldest; - return messages; + return slackApiAllPages("conversations.history", token, params, (data) => data.messages, signal); } async function fetchThreadReplies( @@ -300,13 +302,12 @@ async function buildThreadDocument( channelName: string, replies: SlackMessage[], ): Promise<string> { - const lines: string[] = []; - for (const reply of replies) { + const lines = await mapSequential(replies, async (reply) => { const username = reply.user ? await resolveUser(token, reply.user) : "unknown"; const timestamp = formatTimestamp(reply.ts); const text = reply.text ? convertSlackMrkdwn(await resolveUserMentions(reply.text, token)) : ""; - lines.push(`**${username}** (${timestamp}):\n${text}`); - } + return `**${username}** (${timestamp}):\n${text}`; + }); return `# Thread in #${channelName}\n\n${lines.join("\n\n---\n\n")}`; } @@ -329,44 +330,64 @@ async function fetchAllThreadReplies( userIds: Set<string>, ): Promise<Map<string, SlackMessage[]>> { const threadRepliesMap = new Map<string, SlackMessage[]>(); - for (const threadParent of threadParents) { + await forEachSequential(threadParents, async (threadParent) => { const replies = await fetchThreadReplies(token, channelId, threadParent.ts); threadRepliesMap.set(threadParent.ts, replies); collectUserIds(replies, userIds); - } + }); return threadRepliesMap; } /** Batch-resolve all user IDs that are not already cached. */ async function batchResolveUsers(token: string, userIds: Set<string>): Promise<void> { - for (const userId of userIds) { - if (!userCache.has(userId)) { - await resolveUser(token, userId); - } - } + await forEachSequential( + [...userIds].filter((userId) => !userCache.has(userId)), + (userId) => resolveUser(token, userId), + ); } -/** Index a single standalone message. Returns true if indexed. */ -async function indexStandaloneMessage( +/** + * Index one message with text as its own document, titled + * "<label> — <user>: <start of the text>". + */ +async function indexMessage( db: Database.Database, provider: EmbeddingProvider, token: string, - channelName: string, - msg: SlackMessage, -): Promise<boolean> { - if (!msg.text) return false; - + msg: SlackMessage & { text: string }, + label: string, + url: string, +): Promise<void> { const username = msg.user ? await resolveUser(token, msg.user) : "unknown"; const text = convertSlackMrkdwn(await resolveUserMentions(msg.text, token)); - const title = `#${channelName} — ${username}: ${truncateTitle(msg.text)}`; await indexDocument(db, provider, { - title, + title: `${label} — ${username}: ${truncateTitle(msg.text)}`, content: `**${username}** (${formatTimestamp(msg.ts)}):\n${text}`, sourceType: "manual", - url: `slack://${channelName}/${msg.ts}`, + url, submittedBy: "crawler", }); +} + +/** Index a single standalone message. Returns true if indexed. */ +async function indexStandaloneMessage( + db: Database.Database, + provider: EmbeddingProvider, + token: string, + channelName: string, + msg: SlackMessage, +): Promise<boolean> { + if (!msg.text) return false; + const message = { ...msg, text: msg.text }; + await indexMessage( + db, + provider, + token, + message, + `#${channelName}`, + `slack://${channelName}/${msg.ts}`, + ); return true; } @@ -380,7 +401,7 @@ async function indexThreadsAggregate( threadRepliesMap: Map<string, SlackMessage[]>, ): Promise<number> { let threadsIndexed = 0; - for (const threadParent of threadParents) { + await forEachSequential(threadParents, async (threadParent) => { const replies = threadRepliesMap.get(threadParent.ts) ?? []; const content = await buildThreadDocument(token, channelName, replies); const parentText = threadParent.text ?? ""; @@ -394,7 +415,7 @@ async function indexThreadsAggregate( submittedBy: "crawler", }); threadsIndexed++; - } + }); return threadsIndexed; } @@ -409,26 +430,22 @@ async function indexThreadsSeparate( ): Promise<[number, number]> { let messagesIndexed = 0; let threadsIndexed = 0; - for (const threadParent of threadParents) { + await forEachSequential(threadParents, async (threadParent) => { const replies = threadRepliesMap.get(threadParent.ts) ?? []; - for (const reply of replies) { - if (!reply.text) continue; - - const username = reply.user ? await resolveUser(token, reply.user) : "unknown"; - const text = convertSlackMrkdwn(await resolveUserMentions(reply.text, token)); - const title = `#${channelName} thread reply — ${username}: ${truncateTitle(reply.text)}`; - - await indexDocument(db, provider, { - title, - content: `**${username}** (${formatTimestamp(reply.ts)}):\n${text}`, - sourceType: "manual", - url: `slack://${channelName}/thread/${threadParent.ts}/${reply.ts}`, - submittedBy: "crawler", - }); + await forEachSequential(replies, async (reply) => { + if (!reply.text) return; + await indexMessage( + db, + provider, + token, + { ...reply, text: reply.text }, + `#${channelName} thread reply`, + `slack://${channelName}/thread/${threadParent.ts}/${reply.ts}`, + ); messagesIndexed++; - } + }); threadsIndexed++; - } + }); return [messagesIndexed, threadsIndexed]; } @@ -446,6 +463,17 @@ function classifyMessages(messages: SlackMessage[]): { return { threadParents, standaloneMessages }; } +/** + * Convert a lastSync value to Slack's `oldest` format (Unix seconds). Saved configs store + * lastSync as an ISO-8601 string; values that are already numeric are passed through. + */ +export function toSlackTimestamp(lastSync: string | undefined): string | undefined { + if (!lastSync) return undefined; + if (Number.isFinite(Number(lastSync))) return lastSync; + const ms = Date.parse(lastSync); + return Number.isNaN(ms) ? undefined : String(ms / 1000); +} + /** Process a single channel: fetch messages, resolve users, and index documents. */ async function syncChannel( db: Database.Database, @@ -453,9 +481,10 @@ async function syncChannel( config: SlackConfig, channel: SlackChannel, result: SlackSyncResult, + signal: AbortSignal | undefined, ): Promise<void> { - const oldest = config.lastSync ?? undefined; - const messages = await fetchMessages(config.token, channel.id, oldest); + const oldest = toSlackTimestamp(config.lastSync); + const messages = await fetchMessages(config.token, channel.id, oldest, signal); const { threadParents, standaloneMessages } = classifyMessages(messages); @@ -471,10 +500,11 @@ async function syncChannel( await batchResolveUsers(config.token, allUserIds); - for (const msg of standaloneMessages) { + await forEachSequential(standaloneMessages, async (msg) => { + signal?.throwIfAborted(); const indexed = await indexStandaloneMessage(db, provider, config.token, channel.name, msg); if (indexed) result.messagesIndexed++; - } + }); if (config.threadMode === "aggregate") { result.threadsIndexed += await indexThreadsAggregate( @@ -503,6 +533,27 @@ export async function syncSlack( db: Database.Database, provider: EmbeddingProvider, config: SlackConfig, + options: ConnectorSyncOptions = {}, +): Promise<SlackSyncResult> { + return trackSync( + db, + "slack", + options.syncName ?? "slack", + () => runSlackSync(db, provider, config, options.signal), + (result) => ({ + added: result.messagesIndexed + result.threadsIndexed, + updated: 0, + deleted: 0, + errored: result.errors.length, + }), + ); +} + +async function runSlackSync( + db: Database.Database, + provider: EmbeddingProvider, + config: SlackConfig, + signal: AbortSignal | undefined, ): Promise<SlackSyncResult> { const log = getLogger(); @@ -513,57 +564,45 @@ export async function syncSlack( throw new ValidationError("At least one channel must be specified"); } - const syncId = startSync(db, "slack", "slack"); + userCache.clear(); - try { - userCache.clear(); - - const result: SlackSyncResult = { - channels: 0, - messagesIndexed: 0, - threadsIndexed: 0, - errors: [], - }; - - log.info("Fetching Slack channel list"); - const allChannels = await listChannels(config.token); - const channels = filterChannels(allChannels, config.channels, config.excludeChannels); - result.channels = channels.length; - - log.info({ channelCount: channels.length }, "Processing Slack channels"); - - for (const channel of channels) { - try { - log.info({ channel: channel.name }, "Syncing channel"); - await syncChannel(db, provider, config, channel, result); - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - log.error({ channel: channel.name, err }, "Error syncing Slack channel"); - result.errors.push({ channel: channel.name, error: errMsg }); - } - } + const result: SlackSyncResult = { + channels: 0, + messagesIndexed: 0, + threadsIndexed: 0, + errors: [], + }; - log.info( - { - channels: result.channels, - messages: result.messagesIndexed, - threads: result.threadsIndexed, - }, - "Slack sync complete", - ); + log.info("Fetching Slack channel list"); + const allChannels = await listChannels(config.token, signal); + const channels = filterChannels(allChannels, config.channels, config.excludeChannels); + result.channels = channels.length; + + log.info({ channelCount: channels.length }, "Processing Slack channels"); + + await forEachSequential(channels, async (channel) => { + signal?.throwIfAborted(); + try { + log.info({ channel: channel.name }, "Syncing channel"); + await syncChannel(db, provider, config, channel, result, signal); + } catch (err) { + if (signal?.aborted) throw err; + const errMsg = err instanceof Error ? err.message : String(err); + log.error({ channel: channel.name, err }, "Error syncing Slack channel"); + result.errors.push({ channel: channel.name, error: errMsg }); + } + }); - completeSync(db, syncId, { - added: result.messagesIndexed + result.threadsIndexed, - updated: 0, - deleted: 0, - errored: result.errors.length, - }); + log.info( + { + channels: result.channels, + messages: result.messagesIndexed, + threads: result.threadsIndexed, + }, + "Slack sync complete", + ); - return result; - } catch (err) { - failSync(db, syncId, err instanceof Error ? err.message : String(err)); - throw err; - } + return result; } export function disconnectSlack(db: Database.Database): number { diff --git a/src/connectors/sync-tracker.ts b/src/connectors/sync-tracker.ts index 50e2c4a7..54fca089 100644 --- a/src/connectors/sync-tracker.ts +++ b/src/connectors/sync-tracker.ts @@ -116,3 +116,39 @@ export function getSyncHistory( .prepare(`SELECT * FROM connector_syncs ORDER BY started_at DESC, id DESC LIMIT ?`) .all(limit) as ConnectorSyncRow[]; } + +/** Optional per-call settings accepted by every connector sync function. */ +export interface ConnectorSyncOptions { + /** + * Name recorded in connector_syncs for this run (the saved connector config name). + * Defaults to the connector's historical name when omitted. + */ + syncName?: string | undefined; + /** + * Cancels the sync. Checked between pages and items; an aborted sync throws the + * signal's reason and is recorded as failed. + */ + signal?: AbortSignal | undefined; +} + +/** + * Run `run` and record exactly one connector_syncs row for it: "completed" with the + * stats from `toStats`, or "failed" with the error message (the error is rethrown). + */ +export async function trackSync<R>( + db: Database.Database, + connectorType: string, + connectorName: string, + run: () => Promise<R>, + toStats: (result: R) => SyncStats, +): Promise<R> { + const syncId = startSync(db, connectorType, connectorName); + try { + const result = await run(); + completeSync(db, syncId, toStats(result)); + return result; + } catch (err) { + failSync(db, syncId, err instanceof Error ? err.message : String(err)); + throw err; + } +} diff --git a/src/core/analytics.ts b/src/core/analytics.ts index b2263405..f1bce173 100644 --- a/src/core/analytics.ts +++ b/src/core/analytics.ts @@ -73,7 +73,7 @@ export function logSearch(db: Database.Database, entry: SearchLogEntry): string return id; } -/** Return overview stats for the knowledge base. */ +/** Return overview stats for the knowledge base. `dbPath` defaults to the open database file (`db.name`). */ export function getStats(db: Database.Database, dbPath?: string): OverviewStats { const StatsRowSchema = z.object({ doc_count: z.number(), @@ -99,10 +99,12 @@ export function getStats(db: Database.Database, dbPath?: string): OverviewStats "getStats.row", ); + // Default to the file this connection actually opened. + const sizePath = dbPath ?? (db.memory ? undefined : db.name); let databaseSizeBytes = 0; - if (dbPath) { + if (sizePath) { try { - databaseSizeBytes = statSync(dbPath).size; + databaseSizeBytes = statSync(sizePath).size; } catch { // Inaccessible paths report 0 } diff --git a/src/core/batch-search.ts b/src/core/batch-search.ts deleted file mode 100644 index 7785c18f..00000000 --- a/src/core/batch-search.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { searchDocuments, type SearchOptions, type SearchResponse } from "./search.js"; -import { ValidationError } from "../errors.js"; - -export const BATCH_SEARCH_MAX_REQUESTS = 20; - -export interface BatchSearchRequest { - /** The query string for this search. */ - query: string; - /** Per-request overrides — all SearchOptions except `query` which comes from above. */ - options?: Omit<SearchOptions, "query">; -} - -export interface BatchSearchResponse { - /** Results keyed by the original query string. */ - results: Record<string, SearchResponse>; -} - -/** - * Execute multiple search queries concurrently. - * Results are keyed by the query string. - */ -export async function searchBatch( - db: Database.Database, - provider: EmbeddingProvider, - requests: BatchSearchRequest[], -): Promise<BatchSearchResponse> { - if (!Array.isArray(requests) || requests.length === 0) { - throw new ValidationError("At least one search request is required"); - } - if (requests.length > BATCH_SEARCH_MAX_REQUESTS) { - throw new ValidationError( - `Batch size ${requests.length} exceeds maximum of ${BATCH_SEARCH_MAX_REQUESTS}`, - ); - } - - const entries = await Promise.all( - requests.map(async (req) => { - const searchOpts: SearchOptions = { - ...req.options, - query: req.query, - }; - const response = await searchDocuments(db, provider, searchOpts); - return [req.query, response] as const; - }), - ); - - return { - results: Object.fromEntries(entries), - }; -} diff --git a/src/core/batch.ts b/src/core/batch.ts deleted file mode 100644 index 409f64c1..00000000 --- a/src/core/batch.ts +++ /dev/null @@ -1,138 +0,0 @@ -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { indexDocument } from "./indexing.js"; -import { readFileSync } from "node:fs"; -import { basename } from "node:path"; -import { getLogger } from "../logger.js"; - -export interface BatchImportOptions { - sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; - library?: string | undefined; - version?: string | undefined; - topicId?: string | undefined; - concurrency?: number | undefined; - onProgress?: ((progress: BatchProgress) => void) | undefined; -} - -export interface BatchProgress { - total: number; - completed: number; - failed: number; - currentFile?: string; -} - -export interface BatchFileResult { - file: string; - success: boolean; - chunkCount?: number; - documentId?: string; - error?: string; -} - -export interface BatchImportResult { - total: number; - completed: number; - failed: number; - results: BatchFileResult[]; -} - -/** - * Import multiple files in parallel with configurable concurrency. - * Uses a semaphore pattern to limit concurrent operations. - */ -export async function batchImport( - db: Database.Database, - provider: EmbeddingProvider, - files: string[], - options: BatchImportOptions = {}, -): Promise<BatchImportResult> { - const logger = getLogger(); - const concurrency = options.concurrency ?? 5; - const progress: BatchProgress = { total: files.length, completed: 0, failed: 0 }; - const results: BatchFileResult[] = []; - - logger.debug(`Starting batch import: ${files.length} files, concurrency ${concurrency}`); - - // Semaphore-based parallel processing - let activeCount = 0; - let fileIndex = 0; - - await new Promise<void>((resolve) => { - if (files.length === 0) { - resolve(); - return; - } - - function runNext(): void { - while (activeCount < concurrency && fileIndex < files.length) { - const currentIndex = fileIndex++; - const file = files[currentIndex]!; - activeCount++; - progress.currentFile = file; - - processFile(db, provider, file, options) - .then((result) => { - results[currentIndex] = result; - if (result.success) { - progress.completed++; - } else { - progress.failed++; - } - options.onProgress?.({ ...progress }); - }) - .catch((err) => { - results[currentIndex] = { - file, - success: false, - error: err instanceof Error ? err.message : String(err), - }; - progress.failed++; - options.onProgress?.({ ...progress }); - }) - .finally(() => { - activeCount--; - if (progress.completed + progress.failed === files.length) { - resolve(); - } else { - runNext(); - } - }); - } - } - - runNext(); - }); - - return { - total: files.length, - completed: progress.completed, - failed: progress.failed, - results, - }; -} - -async function processFile( - db: Database.Database, - provider: EmbeddingProvider, - file: string, - options: BatchImportOptions, -): Promise<BatchFileResult> { - const content = readFileSync(file, "utf-8"); - const title = basename(file).replace(/\.[^.]+$/, ""); - - const result = await indexDocument(db, provider, { - title, - content, - sourceType: options.library ? "library" : options.topicId ? "topic" : "manual", - library: options.library, - version: options.version, - topicId: options.topicId, - }); - - return { - file, - success: true, - chunkCount: result.chunkCount, - documentId: result.id, - }; -} diff --git a/src/core/bootstrap.ts b/src/core/bootstrap.ts new file mode 100644 index 00000000..d8b8dd84 --- /dev/null +++ b/src/core/bootstrap.ts @@ -0,0 +1,148 @@ +/** + * One startup sequence for every surface (CLI, MCP server, REST server, SDK, lite): + * load config -> resolve the database path -> open and migrate the database -> + * create the embedding provider -> create or check the vector table. + */ +import type Database from "better-sqlite3"; +import { loadConfig, type LibScopeConfig } from "../config.js"; +import { createDatabase, resolveDatabasePath } from "../db/connection.js"; +import { createVectorTable, runMigrations } from "../db/schema.js"; +import { ConfigError } from "../errors.js"; +import { getLogger } from "../logger.js"; +import { createEmbeddingProvider } from "../providers/index.js"; +import type { EmbeddingProvider } from "../providers/embedding.js"; + +/** Config sections that callers may partially override. */ +export type ConfigOverrides = { [K in keyof LibScopeConfig]?: Partial<LibScopeConfig[K]> }; + +export interface BootstrapOptions { + /** Workspace whose database to open (default: the active workspace). Ignored with `dbPath`. */ + workspace?: string | undefined; + /** Explicit database file. Wins over `workspace` and over `database.path` in config. */ + dbPath?: string | undefined; + /** Use this config instead of loadConfig(). */ + config?: LibScopeConfig | undefined; + /** Per-section values merged over the loaded config (e.g. `{ llm: { model: "x" } }`). */ + configOverrides?: ConfigOverrides | undefined; + /** Use this embedding provider instead of creating one from config. */ + provider?: EmbeddingProvider | undefined; + /** Use an already-open database. It is migrated but not closed by `close()`. */ + db?: Database.Database | undefined; + /** + * Vector table handling: "required" (default) throws on any error; "best-effort" only + * throws when the index belongs to a different embedding model (ConfigError) and otherwise + * continues without vector search; "skip" does not touch it. + */ + vectorTable?: "required" | "best-effort" | "skip" | undefined; + /** + * When false, a provider that cannot be created (e.g. missing API key) does not fail + * startup: commands that do not embed still work, and embedding calls throw the original + * error. Default true. + */ + requireProvider?: boolean | undefined; + /** Where to report the one-time legacy-database notice (default: logger.warn). */ + warn?: ((message: string) => void) | undefined; +} + +export interface Bootstrapped { + db: Database.Database; + provider: EmbeddingProvider; + config: LibScopeConfig; + /** Database file that was opened (":memory:" or the caller's file for an injected db). */ + dbPath: string; + /** Close the database if bootstrap opened it. Safe to call twice. */ + close(): void; +} + +/** Merge per-section overrides over `base` (one level deep). */ +export function mergeConfig(base: LibScopeConfig, overrides?: ConfigOverrides): LibScopeConfig { + if (!overrides) return base; + const merged: Record<string, unknown> = { ...base }; + for (const [section, values] of Object.entries(overrides)) { + if (values === undefined) continue; + const current = (base as unknown as Record<string, unknown>)[section]; + merged[section] = { ...(current as object | undefined), ...values }; + } + return merged as unknown as LibScopeConfig; +} + +/** Stand-in provider used when the configured one cannot be created and it is not required. */ +export class UnavailableEmbeddingProvider implements EmbeddingProvider { + readonly name = "unavailable"; + readonly dimensions = 0; + + constructor(private readonly cause: unknown) {} + + embed(): Promise<number[]> { + return Promise.reject(this.cause instanceof Error ? this.cause : new Error(String(this.cause))); + } + + embedBatch(): Promise<number[][]> { + return Promise.reject(this.cause instanceof Error ? this.cause : new Error(String(this.cause))); + } +} + +function createProvider(config: LibScopeConfig, options: BootstrapOptions): EmbeddingProvider { + if (options.provider) return options.provider; + try { + return createEmbeddingProvider(config); + } catch (err) { + if (options.requireProvider ?? true) throw err; + getLogger().warn({ err }, "Embedding provider unavailable; commands that embed will fail"); + return new UnavailableEmbeddingProvider(err); + } +} + +function setUpVectorTable( + db: Database.Database, + provider: EmbeddingProvider, + mode: NonNullable<BootstrapOptions["vectorTable"]>, +): void { + if (mode === "skip" || provider instanceof UnavailableEmbeddingProvider) return; + try { + createVectorTable(db, provider); + } catch (err) { + if (mode === "required" || err instanceof ConfigError) throw err; + getLogger().warn({ err }, "Vector table unavailable; keyword search only"); + } +} + +/** Open everything a surface needs. Call `close()` when done. */ +export function bootstrap(options: BootstrapOptions = {}): Bootstrapped { + const config = mergeConfig(options.config ?? loadConfig(), options.configOverrides); + + let db: Database.Database; + let dbPath: string; + let ownsDb: boolean; + if (options.db) { + db = options.db; + dbPath = db.name; + ownsDb = false; + } else { + dbPath = resolveDatabasePath({ + explicitPath: options.dbPath ?? config.database.path, + workspace: options.workspace, + warn: options.warn, + }); + db = createDatabase(dbPath); + ownsDb = true; + } + + try { + runMigrations(db); + const provider = createProvider(config, options); + setUpVectorTable(db, provider, options.vectorTable ?? "required"); + return { + db, + provider, + config, + dbPath, + close: (): void => { + if (ownsDb && db.open) db.close(); + }, + }; + } catch (err) { + if (ownsDb) db.close(); + throw err; + } +} diff --git a/src/core/bulk.ts b/src/core/bulk.ts index d00b15e8..a2a97a5c 100644 --- a/src/core/bulk.ts +++ b/src/core/bulk.ts @@ -2,12 +2,7 @@ import type Database from "better-sqlite3"; import { getLogger } from "../logger.js"; import { ValidationError } from "../errors.js"; import { deleteDocument, listDocuments } from "./documents.js"; -import { - addTagsToDocument, - removeTagFromDocument, - getDocumentTags, - getDocumentTagsBatch, -} from "./tags.js"; +import { addTagsToDocument, removeTagsFromDocument, getDocumentTagsBatch } from "./tags.js"; export interface BulkSelector { topicId?: string; @@ -132,14 +127,7 @@ export function bulkRetag( addTagsToDocument(db, id, addTags); } if (removeTags && removeTags.length > 0) { - const docTags = getDocumentTags(db, id); - for (const tagName of removeTags) { - const normalized = tagName.trim().toLowerCase(); - const tag = docTags.find((t) => t.name === normalized); - if (tag) { - removeTagFromDocument(db, id, tag.id); - } - } + removeTagsFromDocument(db, id, removeTags); } } }); diff --git a/src/core/document-view.ts b/src/core/document-view.ts new file mode 100644 index 00000000..64dfc27a --- /dev/null +++ b/src/core/document-view.ts @@ -0,0 +1,55 @@ +import type Database from "better-sqlite3"; +import { getDocument, type Document } from "./documents.js"; +import { getDocumentLinks, type DocumentLinks } from "./links.js"; +import { getDocumentRatings, type RatingSummary } from "./ratings.js"; +import { getDocumentTags } from "./tags.js"; + +export interface DocumentView { + /** The document without its content (see `content`). */ + document: Omit<Document, "content" | "id"> & { documentId: string }; + /** The requested slice of the content. */ + content: string; + /** Length of the full content in characters. */ + contentLength: number; + /** Start of this slice. */ + offset: number; + /** Offset of the next slice, or null when this slice reaches the end. */ + nextOffset: number | null; + tags: string[]; + links: DocumentLinks; + ratings: RatingSummary; +} + +export interface DocumentViewOptions { + /** Character offset to start from (default 0). */ + offset?: number | undefined; + /** Maximum characters of content to return (default: all). */ + maxLength?: number | undefined; +} + +/** + * A document with its tags, links and rating summary, and its content optionally paged + * by character offset. Throws DocumentNotFoundError. + */ +export function getDocumentView( + db: Database.Database, + documentId: string, + options: DocumentViewOptions = {}, +): DocumentView { + const { id, content: full, ...rest } = getDocument(db, documentId); + const offset = Math.min(Math.max(0, options.offset ?? 0), full.length); + const end = + options.maxLength === undefined + ? full.length + : Math.min(full.length, offset + options.maxLength); + return { + document: { documentId: id, ...rest }, + content: full.slice(offset, end), + contentLength: full.length, + offset, + nextOffset: end < full.length ? end : null, + tags: getDocumentTags(db, documentId).map((t) => t.name), + links: getDocumentLinks(db, documentId), + ratings: getDocumentRatings(db, documentId), + }; +} diff --git a/src/core/documents.ts b/src/core/documents.ts index d3ff7e5d..6c312134 100644 --- a/src/core/documents.ts +++ b/src/core/documents.ts @@ -1,10 +1,19 @@ import type Database from "better-sqlite3"; -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { DocumentNotFoundError, ValidationError } from "../errors.js"; -import { chunkContent, chunkContentStreaming, STREAMING_THRESHOLD } from "./indexing.js"; +import { + createChunkWriter, + embedChunks, + embeddingMetaChanged, + splitIntoChunks, + storeDocumentLinks, + type EmbeddingMeta, +} from "./indexing.js"; import { getLogger } from "../logger.js"; import { saveVersion } from "./versioning.js"; +import { emitEvent } from "./events.js"; +import { buildTagFilter } from "./search.js"; export interface Document { id: string; @@ -21,36 +30,28 @@ export interface Document { updatedAt: string; } -/** Get a document by ID. */ -export function getDocument(db: Database.Database, documentId: string): Document { - const row = db - .prepare( - ` - SELECT id, source_type, library, version, topic_id, title, content, url, content_hash, submitted_by, created_at, updated_at - FROM documents WHERE id = ? - `, - ) - .get(documentId) as - | { - id: string; - source_type: string; - library: string | null; - version: string | null; - topic_id: string | null; - title: string; - content: string; - url: string | null; - content_hash: string | null; - submitted_by: string; - created_at: string; - updated_at: string; - } - | undefined; +/** Column list matching {@link DocumentRow}, for `SELECT ... FROM documents`. */ +export const DOC_COLUMNS = + "id, source_type, library, version, topic_id, title, content, url, content_hash, submitted_by, created_at, updated_at"; - if (!row) { - throw new DocumentNotFoundError(documentId); - } +/** Raw `documents` row as returned by a `SELECT ${DOC_COLUMNS}` query. */ +export interface DocumentRow { + id: string; + source_type: string; + library: string | null; + version: string | null; + topic_id: string | null; + title: string; + content: string; + url: string | null; + content_hash: string | null; + submitted_by: string; + created_at: string; + updated_at: string; +} +/** Map a raw `documents` row to a {@link Document}. */ +export function rowToDocument(row: DocumentRow): Document { return { id: row.id, sourceType: row.source_type, @@ -67,17 +68,37 @@ export function getDocument(db: Database.Database, documentId: string): Document }; } +/** + * Delete vector embeddings for every chunk of the given documents. + * Throws on any SQLite error (including a missing `chunk_embeddings` table); + * callers decide how to handle it. + */ +export function deleteChunkEmbeddings(db: Database.Database, docIds: string[]): void { + if (docIds.length === 0) return; + const placeholders = docIds.map(() => "?").join(", "); + db.prepare( + `DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id IN (${placeholders}))`, + ).run(...docIds); +} + +/** Get a document by ID. */ +export function getDocument(db: Database.Database, documentId: string): Document { + const row = db.prepare(`SELECT ${DOC_COLUMNS} FROM documents WHERE id = ?`).get(documentId) as + | DocumentRow + | undefined; + + if (!row) { + throw new DocumentNotFoundError(documentId); + } + + return rowToDocument(row); +} + /** Delete a document and all its chunks/ratings (cascade). */ export function deleteDocument(db: Database.Database, documentId: string): void { // Clean up chunk_embeddings (no foreign key cascade for virtual tables) try { - db.prepare( - ` - DELETE FROM chunk_embeddings WHERE chunk_id IN ( - SELECT id FROM chunks WHERE document_id = ? - ) - `, - ).run(documentId); + deleteChunkEmbeddings(db, [documentId]); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); if (msg.includes("no such table")) { @@ -94,79 +115,80 @@ export function deleteDocument(db: Database.Database, documentId: string): void if (result.changes === 0) { throw new DocumentNotFoundError(documentId); } + emitEvent(db, "document.deleted", { documentId }); } -/** List documents with optional filters. */ -export function listDocuments( - db: Database.Database, - options?: { - library?: string | undefined; - topicId?: string | undefined; - sourceType?: string | undefined; - dateFrom?: string | undefined; - dateTo?: string | undefined; - limit?: number | undefined; - }, -): Document[] { - let sql = ` - SELECT id, source_type, library, version, topic_id, title, content, url, content_hash, submitted_by, created_at, updated_at - FROM documents WHERE 1=1 - `; - const params: unknown[] = []; - - if (options?.library) { - sql += " AND library = ?"; - params.push(options.library); - } - if (options?.topicId) { - sql += " AND topic_id = ?"; - params.push(options.topicId); - } - if (options?.sourceType) { - sql += " AND source_type = ?"; - params.push(options.sourceType); - } - if (options?.dateFrom) { - sql += " AND created_at >= ?"; - params.push(options.dateFrom); +/** Throw DocumentNotFoundError unless a document with this ID exists. */ +export function assertDocumentExists(db: Database.Database, documentId: string): void { + if (!db.prepare("SELECT 1 FROM documents WHERE id = ?").get(documentId)) { + throw new DocumentNotFoundError(documentId); } - if (options?.dateTo) { - sql += " AND created_at <= ?"; - params.push(options.dateTo); +} + +export interface DocumentFilters { + library?: string | undefined; + version?: string | undefined; + topicId?: string | undefined; + sourceType?: string | undefined; + /** Only documents carrying all of these tags. */ + tags?: string[] | undefined; + dateFrom?: string | undefined; + dateTo?: string | undefined; +} + +export interface ListDocumentsOptions extends DocumentFilters { + /** Default 50. */ + limit?: number | undefined; + offset?: number | undefined; +} + +/** WHERE clause (starting with "WHERE 1=1") and params for DocumentFilters on `documents d`. */ +function documentFilterSql(filters: DocumentFilters | undefined): { + where: string; + params: unknown[]; +} { + let where = "WHERE 1=1"; + const params: unknown[] = []; + const equals: Array<[string, string | undefined]> = [ + ["d.library = ?", filters?.library], + ["d.version = ?", filters?.version], + ["d.topic_id = ?", filters?.topicId], + ["d.source_type = ?", filters?.sourceType], + ["d.created_at >= ?", filters?.dateFrom], + ["d.created_at <= ?", filters?.dateTo], + ]; + for (const [clause, value] of equals) { + if (value) { + where += ` AND ${clause}`; + params.push(value); + } } + const tagFilter = buildTagFilter(filters?.tags, "d"); + return { where: where + tagFilter.clause, params: [...params, ...tagFilter.params] }; +} - sql += " ORDER BY updated_at DESC LIMIT ?"; - params.push(options?.limit ?? 50); - - const rows = db.prepare(sql).all(...params) as Array<{ - id: string; - source_type: string; - library: string | null; - version: string | null; - topic_id: string | null; - title: string; - content: string; - url: string | null; - content_hash: string | null; - submitted_by: string; - created_at: string; - updated_at: string; - }>; - - return rows.map((row) => ({ - id: row.id, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - title: row.title, - content: row.content, - url: row.url, - contentHash: row.content_hash, - submittedBy: row.submitted_by, - createdAt: row.created_at, - updatedAt: row.updated_at, - })); +/** List documents with optional filters, most recently updated first. */ +export function listDocuments(db: Database.Database, options?: ListDocumentsOptions): Document[] { + const { where, params } = documentFilterSql(options); + const columns = DOC_COLUMNS.split(", ") + .map((c) => `d.${c}`) + .join(", "); + const rows = db + .prepare( + `SELECT ${columns} FROM documents d ${where} ORDER BY d.updated_at DESC LIMIT ? OFFSET ?`, + ) + .all(...params, options?.limit ?? 50, options?.offset ?? 0) as DocumentRow[]; + + return rows.map((row) => rowToDocument(row)); +} + +/** Number of documents matching the filters (for paging listDocuments). */ +export function countDocuments(db: Database.Database, filters?: DocumentFilters): number { + const { where, params } = documentFilterSql(filters); + const row = db.prepare(`SELECT COUNT(*) AS n FROM documents d ${where}`).get(...params) as { + n: number; + }; + return row.n; } export interface UpdateDocumentInput { @@ -220,20 +242,20 @@ export async function updateDocument( // SQLite's datetime('now') uses the OS clock and cannot be mocked in unit tests. const updatedAt = new Date().toISOString().replace("T", " ").slice(0, 19); + const newMeta = { title: newTitle, library: newLibrary, version: newVersion }; + if (contentChanged) { log.info({ docId: documentId }, "Content changed, re-chunking and re-indexing embeddings"); - const useStreaming = newContent.length > STREAMING_THRESHOLD; - const chunks = useStreaming ? chunkContentStreaming(newContent) : chunkContent(newContent); - const embeddings = await provider.embedBatch(chunks); + const chunks = splitIntoChunks(newContent); + const embeddings = await embedChunks(provider, chunks, newMeta); + const writer = createChunkWriter(db); const transaction = db.transaction(() => { saveVersion(db, documentId); try { - db.prepare( - "DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)", - ).run(documentId); + deleteChunkEmbeddings(db, [documentId]); } catch (err: unknown) { // chunk_embeddings table may not exist log.debug({ err, documentId }, "Skipped chunk_embeddings cleanup during update"); @@ -255,39 +277,62 @@ export async function updateDocument( documentId, ); - const insertChunk = db.prepare( - "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", - ); - const insertEmbedding = db.prepare( - "INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)", - ); - - for (let i = 0; i < chunks.length; i++) { - const chunkId = randomUUID(); - insertChunk.run(chunkId, documentId, chunks[i] ?? "", i); - - try { - const vecBuffer = Buffer.from(new Float32Array(embeddings[i] ?? []).buffer); - insertEmbedding.run(chunkId, vecBuffer); - } catch (err: unknown) { - // chunk_embeddings table may not exist - log.debug({ err, chunkId }, "Skipped embedding insertion during update"); - } - } + writer.insertChunks(documentId, chunks, embeddings); }); transaction(); + storeDocumentLinks(db, documentId, newContent); } else { + // Title/library/version are part of the embedding text, so re-embed existing chunks + // (keeping their IDs) when they change. + const reembed = embeddingMetaChanged(existing, newMeta) + ? await embedExistingChunks(db, provider, documentId, newMeta) + : null; + const transaction = db.transaction(() => { saveVersion(db, documentId); db.prepare( `UPDATE documents SET title = ?, library = ?, version = ?, url = ?, topic_id = ?, updated_at = ? WHERE id = ?`, ).run(newTitle, newLibrary, newVersion, newUrl, newTopicId, updatedAt, documentId); + + reembed?.write(); }); transaction(); } + emitEvent(db, "document.updated", { + documentId, + title: newTitle, + library: newLibrary, + version: newVersion, + }); return getDocument(db, documentId); } + +/** + * Embed a document's existing chunks with new metadata. Returns a `write()` step that + * replaces the stored vectors; call it inside the caller's transaction. + */ +async function embedExistingChunks( + db: Database.Database, + provider: EmbeddingProvider, + documentId: string, + meta: EmbeddingMeta, +): Promise<{ write: () => void }> { + const rows = db + .prepare("SELECT id, content FROM chunks WHERE document_id = ? ORDER BY chunk_index") + .all(documentId) as Array<{ id: string; content: string }>; + const embeddings = await embedChunks( + provider, + rows.map((r) => r.content), + meta, + ); + const writer = createChunkWriter(db); + return { + write: (): void => { + rows.forEach((row, i) => writer.replaceEmbedding(row.id, embeddings[i] ?? [])); + }, + }; +} diff --git a/src/core/events.ts b/src/core/events.ts new file mode 100644 index 00000000..b1c3f6dc --- /dev/null +++ b/src/core/events.ts @@ -0,0 +1,58 @@ +import type Database from "better-sqlite3"; +import { getLogger } from "../logger.js"; + +/** Events emitted by core mutation functions. Webhooks subscribe to these. */ +export const LIBSCOPE_EVENTS = [ + "document.created", + "document.updated", + "document.deleted", + "document.rated", + "search.executed", +] as const; + +export type LibScopeEvent = (typeof LIBSCOPE_EVENTS)[number]; + +/** + * A listener receives every emitted event. It runs synchronously inside the emitting call, + * so it must start any slow work (network I/O) without awaiting it. + */ +export type EventListener = ( + db: Database.Database, + event: LibScopeEvent, + data: Record<string, unknown>, +) => void | Promise<void>; + +const listeners = new Set<EventListener>(); + +/** Subscribe to all events. Returns a function that removes the listener. */ +export function onEvent(listener: EventListener): () => void { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +function logListenerError(err: unknown, event: LibScopeEvent): void { + getLogger().warn({ err, event }, "Event listener failed"); +} + +/** + * Emit an event to all listeners. Never throws into the caller and never waits for + * asynchronous listener work; listener errors and rejections are logged. + */ +export function emitEvent( + db: Database.Database, + event: LibScopeEvent, + data: Record<string, unknown>, +): void { + for (const listener of listeners) { + try { + const pending = listener(db, event, data); + if (pending instanceof Promise) { + pending.catch((err: unknown) => logListenerError(err, event)); + } + } catch (err) { + logListenerError(err, event); + } + } +} diff --git a/src/core/graph.ts b/src/core/graph.ts index 37613031..85c215b5 100644 --- a/src/core/graph.ts +++ b/src/core/graph.ts @@ -187,17 +187,17 @@ function buildTagNodesAndEdges( docIds: Set<string>, ): { nodes: GraphNode[]; edges: GraphEdge[] } { const tags = db.prepare("SELECT id, name FROM tags ORDER BY name").all() as TagRow[]; - const tagIdMap = new Map<string, string>(); + const tagIds = new Set<string>(); const nodes: GraphNode[] = []; for (const tag of tags) { - tagIdMap.set(tag.id, tag.name); + tagIds.add(tag.id); nodes.push({ id: `tag:${tag.id}`, label: tag.name, type: "tag", metadata: {} }); } const docTags = db.prepare("SELECT document_id, tag_id FROM document_tags").all() as DocTagRow[]; const edges: GraphEdge[] = []; for (const dt of docTags) { - if (docIds.has(dt.document_id) && tagIdMap.has(dt.tag_id)) { + if (docIds.has(dt.document_id) && tagIds.has(dt.tag_id)) { edges.push({ source: dt.document_id, target: `tag:${dt.tag_id}`, diff --git a/src/core/html-entities.ts b/src/core/html-entities.ts new file mode 100644 index 00000000..88ed5f1d --- /dev/null +++ b/src/core/html-entities.ts @@ -0,0 +1,42 @@ +/** Decoding of HTML character references in text taken from raw HTML. */ + +const NAMED_ENTITIES: Readonly<Record<string, string>> = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: " ", + ndash: "–", + mdash: "—", + hellip: "…", + lsquo: "‘", + rsquo: "’", + ldquo: "“", + rdquo: "”", + laquo: "«", + raquo: "»", + middot: "·", + bull: "•", + copy: "©", + reg: "®", + trade: "™", +}; + +/** + * Decode HTML character references in text taken from raw HTML (for example a page title): + * numeric references (`—`, `—`) and common named ones (`&`, `—`). + * Unknown or invalid references are kept as they are. + */ +export function decodeHtmlEntities(text: string): string { + return text.replaceAll( + /&(#\d{1,7}|#x[\da-f]{1,6}|[a-z]{2,8});/gi, + (ref: string, body: string) => { + if (!body.startsWith("#")) return NAMED_ENTITIES[body.toLowerCase()] ?? ref; + const hex = body[1] === "x" || body[1] === "X"; + const codePoint = Number.parseInt(body.slice(hex ? 2 : 1), hex ? 16 : 10); + if (codePoint === 0 || codePoint > 0x10ffff) return ref; + return String.fromCodePoint(codePoint); + }, + ); +} diff --git a/src/core/index.ts b/src/core/index.ts index 1e956129..d9e97cf6 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -1,255 +1,58 @@ -export { indexDocument, indexFile, chunkContent } from "./indexing.js"; -export type { - IndexDocumentInput, - IndexFileOptions, - IndexedDocument, - ChunkOptions, -} from "./indexing.js"; - -export { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; -export type { DocumentParser } from "./parsers/index.js"; - -export { checkDuplicate, findDuplicates } from "./dedup.js"; -export type { DedupResult, DedupOptions, DuplicateGroup } from "./dedup.js"; - -export { searchDocuments } from "./search.js"; -export type { - SearchOptions, - SearchResult, - SearchMethod, - ScoreExplanation, - ContextChunk, -} from "./search.js"; - -export { - logSearch, - recordSearchQuery, - getStats, - getPopularDocuments, - getStaleDocuments, - getTopQueries, - getSearchTrends, - getSearchAnalytics, - getKnowledgeGaps, -} from "./analytics.js"; +/** + * Package root (`import ... from "libscope"`): the LibScope class, its option and result + * types, and the error classes. Everything else is reached through LibScope methods. + * tests/unit/root-exports.test.ts lists the runtime exports so additions are deliberate. + */ +export { LibScope } from "../LibScope.js"; export type { - SearchLogEntry, - RecordSearchQueryInput, - OverviewStats, - PopularDocument, - StaleDocument, - TopQuery, - SearchTrend, - SearchAnalytics, - KnowledgeGap, -} from "./analytics.js"; - -export { rateDocument, getDocumentRatings, listRatings } from "./ratings.js"; -export type { RateDocumentInput, Rating, RatingSummary } from "./ratings.js"; - -export { getDocument, deleteDocument, listDocuments, updateDocument } from "./documents.js"; -export type { Document, UpdateDocumentInput } from "./documents.js"; - -export { - saveVersion, - getVersionHistory, - getVersion, - rollbackToVersion, - pruneVersions, - MAX_VERSIONS_DEFAULT, -} from "./versioning.js"; -export type { DocumentVersion } from "./versioning.js"; - -export { - createTopic, - listTopics, - getTopic, - deleteTopic, - renameTopic, - getDocumentsByTopic, - getTopicStats, -} from "./topics.js"; -export type { Topic, CreateTopicInput, GetDocumentsByTopicOptions, TopicStats } from "./topics.js"; - -export { fetchAndConvert, DEFAULT_FETCH_OPTIONS } from "./url-fetcher.js"; -export type { FetchedDocument, FetchOptions } from "./url-fetcher.js"; - -export { exportKnowledgeBase, importFromBackup } from "./export.js"; - -export { batchImport } from "./batch.js"; + LibScopeOptions, + RunOptions, + NamespaceName, + LibScopeNamespace, + LibScopeInput, + LibScopeOutput, + OperationInput, + OperationOutput, + AddInput, + AddResult, + SearchInput, + SearchOutput, + AskInput, + AskOutput, +} from "../LibScope.js"; + +export { + LibScopeError, + DatabaseError, + EmbeddingError, + ValidationError, + FetchError, + ConfigError, + NotFoundError, + DocumentNotFoundError, + ChunkNotFoundError, + TopicNotFoundError, +} from "../errors.js"; + +// Providers and hooks callers can pass to LibScope.create. +export type { EmbeddingProvider } from "../providers/embedding.js"; +export type { LlmProvider } from "./rag.js"; +export type { Chunker } from "./indexing.js"; +export type { ConfigOverrides } from "./bootstrap.js"; +export type { LibScopeConfig } from "../config.js"; + +// Result element types. +export type { ListResult, ProgressEvent } from "./operations/types.js"; +export type { DocumentSummary } from "./operations/documents.js"; +export type { SearchResult, ContextChunk, ScoreExplanation } from "./search.js"; export type { - BatchImportOptions, - BatchImportResult, - BatchProgress, - BatchFileResult, -} from "./batch.js"; - -export { - createTag, - deleteTag, - listTags, - addTagsToDocument, - removeTagFromDocument, - getDocumentTags, - getDocumentsByTag, - suggestTags, -} from "./tags.js"; -export type { Tag, TagWithCount, GetDocumentsByTagOptions } from "./tags.js"; - -export { FileWatcher, DEFAULT_WATCH_EXTENSIONS } from "./watcher.js"; -export type { WatchOptions } from "./watcher.js"; - -export { - askQuestion, - askQuestionStream, - createLlmProvider, - buildContextPrompt, - extractSources, + AnswerResult, + PassthroughResult, + RagResult, + RagSource, + RagStreamEvent, } from "./rag.js"; -export type { RagOptions, RagResult, RagSource, RagStreamEvent, LlmProvider } from "./rag.js"; - -export { reindex } from "./reindex.js"; -export type { ReindexOptions, ReindexResult, ReindexProgress } from "./reindex.js"; - -export { - listAvailablePacks, - installPack, - removePack, - listInstalledPacks, - createPack, -} from "./packs.js"; -export type { - KnowledgePack, - PackDocument, - PackInfo, - InstalledPack, - InstallResult, - CreatePackOptions, -} from "./packs.js"; - -export { indexRepository, parseRepoUrl } from "./repo.js"; -export type { RepoOptions, RepoResult } from "./repo.js"; - -export { - syncOneNote, - disconnectOneNote, - authenticateDeviceCode, - refreshAccessToken, - convertOneNoteHtml, -} from "../connectors/onenote.js"; -export type { OneNoteConfig, OneNoteSyncResult } from "../connectors/onenote.js"; - -export { - syncObsidianVault, - parseObsidianMarkdown, - disconnectVault, -} from "../connectors/obsidian.js"; -export type { ObsidianConfig, SyncResult } from "../connectors/obsidian.js"; - -export { registerProvider, createEmbeddingProvider } from "../providers/index.js"; -export type { EmbeddingProvider } from "../providers/embedding.js"; -export type { ProviderFactory } from "../providers/index.js"; - -export { - createWorkspace, - deleteWorkspace, - listWorkspaces, - getWorkspacePath, - getWorkspacesDir, - getActiveWorkspace, - setActiveWorkspace, - DEFAULT_WORKSPACE, -} from "./workspace.js"; -export type { Workspace } from "./workspace.js"; - -export { buildKnowledgeGraph, detectClusters } from "./graph.js"; -export type { KnowledgeGraph, GraphNode, GraphEdge, GraphOptions } from "./graph.js"; - -export { - createLink, - getDocumentLinks, - deleteLink, - getPrerequisiteChain, - listLinks, -} from "./links.js"; -export type { LinkType, DocumentLink, DocumentLinkWithTitle, DocumentLinks } from "./links.js"; - -export { startApiServer } from "../api/server.js"; -export type { ApiServerOptions } from "../api/server.js"; - -export { syncNotion, convertNotionBlocks, disconnectNotion } from "../connectors/notion.js"; -export type { NotionConfig, NotionSyncResult, NotionBlock } from "../connectors/notion.js"; - -export { syncSlack, convertSlackMrkdwn, disconnectSlack } from "../connectors/slack.js"; -export type { SlackConfig, SlackSyncResult } from "../connectors/slack.js"; - -export { - saveDbConnectorConfig, - loadDbConnectorConfig, - deleteDbConnectorConfig, - loadConnectorConfig, - saveConnectorConfig, - saveNamedConnectorConfig, - loadNamedConnectorConfig, - hasNamedConnectorConfig, - deleteConnectorDocuments, -} from "../connectors/index.js"; -export type { ConnectorConfig } from "../connectors/index.js"; - -export { - createSavedSearch, - listSavedSearches, - getSavedSearch, - deleteSavedSearch, - runSavedSearch, -} from "./saved-searches.js"; -export type { SavedSearch } from "./saved-searches.js"; - -export { - syncConfluence, - convertConfluenceStorage, - disconnectConfluence, -} from "../connectors/confluence.js"; -export type { ConfluenceConfig, ConfluenceSyncResult } from "../connectors/confluence.js"; - -export { - syncDocSite, - disconnectDocSite, - detectDocSiteType, - extractDocLinks, - extractDocTitle, - extractMainContent, - extractElementByPattern, - extractSitemapUrls, - normalizeUrl as normalizeDocUrl, -} from "../connectors/docs.js"; -export type { DocSiteConfig, DocSiteSyncResult, DocSiteType } from "../connectors/docs.js"; - -export { resolveSelector, bulkDelete, bulkRetag, bulkMove } from "./bulk.js"; -export type { BulkSelector, BulkResult } from "./bulk.js"; - -export { - createWebhook, - listWebhooks, - getWebhook, - deleteWebhook, - updateWebhook, - signPayload, - buildPayload, - fireWebhooks, - redactWebhook, - WEBHOOK_EVENTS, -} from "./webhooks.js"; -export type { Webhook, RedactedWebhook, WebhookEvent, WebhookPayload } from "./webhooks.js"; - -export { ConnectorScheduler, loadScheduleEntries } from "./scheduler.js"; -export type { ScheduleConfig, SchedulerStatus } from "./scheduler.js"; - -export { searchBatch, BATCH_SEARCH_MAX_REQUESTS } from "./batch-search.js"; -export type { BatchSearchRequest, BatchSearchResponse } from "./batch-search.js"; - -export { pruneExpiredDocuments } from "./ttl.js"; -export type { PruneResult } from "./ttl.js"; - -export { LibScope } from "../LibScope.js"; -export type { LibScopeOptions } from "../LibScope.js"; +export type { IngestResult, IngestedDocument, IngestKind } from "./ingest.js"; +export type { DocumentView } from "./document-view.js"; +export type { Overview } from "./overview.js"; +export type { Task, TaskStatus } from "./tasks.js"; diff --git a/src/core/indexing.ts b/src/core/indexing.ts index 03641e4d..a140a972 100644 --- a/src/core/indexing.ts +++ b/src/core/indexing.ts @@ -4,17 +4,20 @@ import { readFileSync } from "node:fs"; import { basename } from "node:path"; import type { Readable } from "node:stream"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { ValidationError } from "../errors.js"; +import { EmbeddingError, ValidationError } from "../errors.js"; +import { REBUILD_VECTOR_INDEX_HINT, isVectorDimensionError } from "../db/index-meta.js"; import { getLogger } from "../logger.js"; import { checkDuplicate } from "./dedup.js"; import type { DedupOptions } from "./dedup.js"; +import { deleteChunkEmbeddings } from "./documents.js"; import { extractAndStoreDocumentLinks } from "./links.js"; +import { emitEvent } from "./events.js"; import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; export interface IndexDocumentInput { title: string; content: string; - sourceType: "library" | "topic" | "manual" | "model-generated"; + sourceType: SourceType; library?: string | undefined; version?: string | undefined; topicId?: string | undefined; @@ -334,9 +337,7 @@ function handleUrlDedup( log.info({ docId: existing.id, url }, "Document updated, re-indexing"); try { - db.prepare( - "DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)", - ).run(existing.id); + deleteChunkEmbeddings(db, [existing.id]); } catch (err: unknown) { log.debug({ err, docId: existing.id }, "Skipped chunk_embeddings cleanup during re-index"); } @@ -375,36 +376,109 @@ function handleTitleLengthDedup( ); } +/** Document fields that are folded into every chunk's embedding text. */ +export interface EmbeddingMeta { + title?: string | null | undefined; + library?: string | null | undefined; + version?: string | null | undefined; +} + /** Build the metadata prefix for embedding enrichment. */ -function buildMetaPrefix(input: IndexDocumentInput): string { +function buildMetaPrefix(meta: EmbeddingMeta): string { const parts: string[] = []; - if (input.title) parts.push(input.title); - if (input.library) parts.push(`Library: ${input.library}`); - if (input.version) parts.push(`Version: ${input.version}`); + if (meta.title) parts.push(meta.title); + if (meta.library) parts.push(`Library: ${meta.library}`); + if (meta.version) parts.push(`Version: ${meta.version}`); return parts.length > 0 ? parts.join(" | ") + "\n\n" : ""; } -/** Try to prepare an insert statement for chunk_embedding_metadata if the table exists. */ -function tryPrepareMetaInsert( - db: Database.Database, -): Database.Statement<[string, string, string]> | null { +/** + * The text that is embedded for a chunk: a title/library/version prefix plus the chunk. + * Every write path (index, update, reindex, pack install) must embed this text so that + * a chunk's vector does not depend on how its document was written. + */ +export function buildEmbeddingText(chunk: string, meta: EmbeddingMeta): string { + return buildMetaPrefix(meta) + chunk; +} + +/** True when two documents' metadata would produce different embedding text. */ +export function embeddingMetaChanged(a: EmbeddingMeta, b: EmbeddingMeta): boolean { + return buildMetaPrefix(a) !== buildMetaPrefix(b); +} + +/** Split document content into chunks, using windowed chunking for very large documents. */ +export function splitIntoChunks(content: string): string[] { + return content.length > STREAMING_THRESHOLD + ? chunkContentStreaming(content) + : chunkContent(content); +} + +/** Embed a document's chunks using {@link buildEmbeddingText}. */ +export function embedChunks( + provider: EmbeddingProvider, + chunks: string[], + meta: EmbeddingMeta, +): Promise<number[][]> { + return provider.embedBatch(chunks.map((c) => buildEmbeddingText(c, meta))); +} + +function isNoSuchTableError(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return message.includes("no such table"); +} + +/** Writes chunk rows and vectors the same way for every write path. */ +export interface ChunkWriter { + /** Insert chunk rows for a document with their embeddings (matched by array index). */ + insertChunks(documentId: string, chunks: string[], embeddings: ReadonlyArray<number[]>): void; + /** Replace the stored embedding of an existing chunk. */ + replaceEmbedding(chunkId: string, embedding: number[]): void; +} + +/** + * Create a {@link ChunkWriter}. Writes are synchronous; callers wrap them in a transaction. + * When the vector table does not exist (sqlite-vec not loaded), vectors are skipped. + */ +export function createChunkWriter(db: Database.Database): ChunkWriter { const log = getLogger(); - try { - const exists = db - .prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name='chunk_embedding_metadata'", - ) - .get(); - if (exists) { - return db.prepare(` - INSERT OR REPLACE INTO chunk_embedding_metadata (chunk_id, embedding_provider, embedding_model) - VALUES (?, ?, ?) - `); + const insertChunk = db.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", + ); + let insertVec: Database.Statement | undefined; + let deleteVec: Database.Statement | undefined; + + const writeVector = (chunkId: string, embedding: number[], replace: boolean): void => { + try { + if (replace) { + deleteVec ??= db.prepare("DELETE FROM chunk_embeddings WHERE chunk_id = ?"); + deleteVec.run(chunkId); + } + insertVec ??= db.prepare("INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)"); + insertVec.run(chunkId, Buffer.from(new Float32Array(embedding).buffer)); + } catch (err) { + if (isVectorDimensionError(err)) { + throw new EmbeddingError( + `The embedding has ${embedding.length} dimensions, which does not match the vector index. ${REBUILD_VECTOR_INDEX_HINT}`, + err, + ); + } + if (!isNoSuchTableError(err)) throw err; + log.debug({ chunkId }, "Skipped vector insertion (sqlite-vec not loaded)"); } - } catch (err: unknown) { - log.debug({ err }, "Skipped chunk_embedding_metadata check"); - } - return null; + }; + + return { + insertChunks(documentId, chunks, embeddings): void { + for (let i = 0; i < chunks.length; i++) { + const chunkId = randomUUID(); + insertChunk.run(chunkId, documentId, chunks[i] ?? "", i); + writeVector(chunkId, embeddings[i] ?? [], false); + } + }, + replaceEmbedding(chunkId, embedding): void { + writeVector(chunkId, embedding, true); + }, + }; } /** Index a document: validate, chunk, embed, and store. */ @@ -432,33 +506,20 @@ export async function indexDocument( if (titleResult) return titleResult; const docId = randomUUID(); - let chunks: string[]; - if (input.preChunked && input.preChunked.length > 0) { - chunks = input.preChunked; - } else { - const useStreaming = input.content.length > STREAMING_THRESHOLD; - chunks = useStreaming ? chunkContentStreaming(input.content) : chunkContent(input.content); - } + const chunks = + input.preChunked && input.preChunked.length > 0 + ? input.preChunked + : splitIntoChunks(input.content); log.info({ docId, title: input.title, chunkCount: chunks.length }, "Indexing document"); - const metaPrefix = buildMetaPrefix(input); - const textsForEmbedding = chunks.map((c) => metaPrefix + c); - const embeddings = await provider.embedBatch(textsForEmbedding); + const embeddings = await embedChunks(provider, chunks, input); const insertDoc = db.prepare(` INSERT INTO documents (id, source_type, library, version, topic_id, title, content, url, submitted_by, content_hash, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); - const insertChunk = db.prepare(` - INSERT INTO chunks (id, document_id, content, chunk_index) - VALUES (?, ?, ?, ?) - `); - const insertEmbedding = db.prepare(` - INSERT INTO chunk_embeddings (chunk_id, embedding) - VALUES (?, ?) - `); - const insertMeta = tryPrepareMetaInsert(db); + const writer = createChunkWriter(db); const transaction = db.transaction(() => { insertDoc.run( @@ -474,32 +535,30 @@ export async function indexDocument( contentHash, input.expiresAt ?? null, ); - - for (let i = 0; i < chunks.length; i++) { - const chunkId = randomUUID(); - insertChunk.run(chunkId, docId, chunks[i] ?? "", i); - try { - const vecBuffer = Buffer.from(new Float32Array(embeddings[i] ?? []).buffer); - insertEmbedding.run(chunkId, vecBuffer); - insertMeta?.run(chunkId, provider.name, "unknown"); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - if (!message.includes("no such table")) throw err; - log.debug({ chunkId }, "Skipped vector insertion (sqlite-vec not loaded)"); - } - } + writer.insertChunks(docId, chunks, embeddings); }); transaction(); log.info({ docId, chunkCount: chunks.length }, "Document indexed successfully"); + storeDocumentLinks(db, docId, input.content); + + emitEvent(db, "document.created", { + documentId: docId, + title: input.title, + library: input.library, + version: input.version, + }); + return { id: docId, chunkCount: chunks.length }; +} + +/** Extract markdown/wiki links from content and store them; failures are logged, not thrown. */ +export function storeDocumentLinks(db: Database.Database, docId: string, content: string): void { try { - extractAndStoreDocumentLinks(db, docId, input.content); + extractAndStoreDocumentLinks(db, docId, content); } catch (err) { - log.warn({ err, docId }, "Failed to extract document links"); + getLogger().warn({ err, docId }, "Failed to extract document links"); } - - return { id: docId, chunkCount: chunks.length }; } export interface IndexFileOptions { @@ -509,6 +568,48 @@ export interface IndexFileOptions { title?: string | undefined; format?: string | undefined; dedup?: "skip" | "warn" | "force" | undefined; + /** Overrides the source type derived from `library`/`topic`. */ + sourceType?: IndexDocumentInput["sourceType"] | undefined; + /** ISO 8601 expiry timestamp (see IndexDocumentInput.expiresAt). */ + expiresAt?: string | undefined; + /** Custom chunker; see {@link Chunker}. */ + chunker?: Chunker | undefined; + /** Stored as the document URL; re-indexing a file with the same URL updates that document. */ + url?: string | undefined; +} + +/** + * Splits a document into chunks instead of the built-in markdown chunker. `source` is the + * file path or URL the content came from ("" for inline content). Return undefined to use + * the built-in chunker for this document. + */ +export type Chunker = (doc: { + content: string; + title: string; + source: string; +}) => Promise<string[] | undefined> | string[] | undefined; + +/** Document source types. */ +export const SOURCE_TYPES = ["library", "topic", "manual", "model-generated"] as const; +export type SourceType = (typeof SOURCE_TYPES)[number]; + +/** + * The one rule for a document's default source type, used by every write path + * (indexFile, ingest, and through ingest the CLI, MCP, REST and SDK): + * 1. an explicit `sourceType` wins; + * 2. else "library" when a library name is given; + * 3. else "topic" when a topic is given; + * 4. else "manual". + * "model-generated" is never inferred; callers set it explicitly. + */ +export function resolveSourceType(input: { + sourceType?: SourceType | undefined; + library?: string | undefined; + topic?: string | undefined; +}): SourceType { + if (input.sourceType) return input.sourceType; + if (input.library) return "library"; + return input.topic ? "topic" : "manual"; } /** @@ -529,27 +630,31 @@ export async function indexFile( : undefined; const effectiveName = normalizedFormat ? `file${normalizedFormat}` : filePath; const parser = getParserForFile(effectiveName); - - if (!parser) { - const supported = getSupportedExtensions().join(", "); - throw new ValidationError( - `Unsupported file format: "${filePath}". Supported extensions: ${supported}`, + const unsupported = (): ValidationError => + new ValidationError( + `Unsupported file format: "${filePath}". Supported extensions: ${getSupportedExtensions().join(", ")}`, ); - } + // Without a parser, a custom chunker may still take the file as UTF-8 text (e.g. source code). + if (!parser && !options.chunker) throw unsupported(); - log.info({ filePath, parser: parser.extensions[0] }, "Parsing file for indexing"); + log.info({ filePath, parser: parser?.extensions[0] ?? "chunker" }, "Parsing file for indexing"); const buffer = readFileSync(filePath); - const content = await parser.parse(buffer); + const content = parser ? await parser.parse(buffer) : buffer.toString("utf-8"); const title = options.title ?? basename(filePath).replace(/\.[^.]+$/, ""); + const preChunked = await options.chunker?.({ content, title, source: filePath }); + if (!parser && !preChunked) throw unsupported(); return indexDocument(db, provider, { title, content, - sourceType: options.library ? "library" : options.topic ? "topic" : "manual", + sourceType: resolveSourceType(options), library: options.library, version: options.version, topicId: options.topic, dedup: options.dedup, + expiresAt: options.expiresAt, + preChunked, + url: options.url, }); } diff --git a/src/core/ingest.ts b/src/core/ingest.ts new file mode 100644 index 00000000..1b40e993 --- /dev/null +++ b/src/core/ingest.ts @@ -0,0 +1,410 @@ +/** + * One entry point for "add this to the knowledge base": inline content, a local file, a + * directory (with include/exclude globs), a single URL, a URL crawl (spider), or a GitHub / + * GitLab repository. The CLI `add`, MCP `submit-document`, REST document routes and the SDK + * all reach indexing through here (via the `add` operation), so they share one set of + * defaults. The default source type comes from {@link resolveSourceType}. + */ +import type Database from "better-sqlite3"; +import { readdirSync, statSync } from "node:fs"; +import { basename, join, resolve } from "node:path"; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import { ValidationError } from "../errors.js"; +import { forEachSequential } from "../utils/async.js"; +import { globToRegExp, toPosixPath } from "../utils/glob.js"; +import { + indexDocument, + indexFile, + resolveSourceType, + type Chunker, + type IndexDocumentInput, + type IndexedDocument, + type SourceType, +} from "./indexing.js"; +import { getParserForFile } from "./parsers/index.js"; +import { fetchAndConvert, type FetchOptions } from "./url-fetcher.js"; +import { spiderUrl, type SpiderOptions, type SpiderStats } from "./spider.js"; +import { indexRepository, isRepoUrl } from "./repo.js"; +import { addTagsToDocument } from "./tags.js"; + +export type IngestKind = "content" | "file" | "directory" | "url" | "repo"; + +export interface IngestInput { + /** Local file, directory, URL or repository URL. Omit when `content` is given. */ + source?: string | undefined; + /** How to read `source`. Default "auto": see {@link detectIngestKind}. */ + kind?: IngestKind | "auto" | undefined; + /** Inline document content (markdown). */ + content?: string | undefined; + /** Title. Required with `content`; overrides the detected title for a file or single URL. */ + title?: string | undefined; + /** Canonical URL stored with inline content. */ + url?: string | undefined; + topic?: string | undefined; + library?: string | undefined; + version?: string | undefined; + sourceType?: SourceType | undefined; + /** Tags added to every indexed document. */ + tags?: string[] | undefined; + /** ISO 8601 expiry timestamp; expired documents are removed by pruneExpiredDocuments(). */ + expiresAt?: string | undefined; + dedup?: "skip" | "warn" | "force" | undefined; + /** Force a file format (e.g. ".pdf") instead of using the extension. */ + format?: string | undefined; + /** Directory: globs a file's relative path must match (default: every supported format). */ + include?: string[] | undefined; + /** Directory: globs for files to skip. A glob without "/" also matches the file name. */ + exclude?: string[] | undefined; + /** URL: crawl pages linked from the URL. */ + spider?: boolean | undefined; + maxPages?: number | undefined; + maxDepth?: number | undefined; + sameDomain?: boolean | undefined; + pathPrefix?: string | undefined; + /** URL crawl: globs for URLs to skip. */ + excludePatterns?: string[] | undefined; + /** Repository: branch (default: from the URL, else "main"). */ + branch?: string | undefined; + /** Repository: only index files under these paths. */ + paths?: string[] | undefined; + /** Repository: file extensions to index (default: .md, .mdx, .txt, .rst). */ + extensions?: string[] | undefined; + /** Repository: access token for private repositories. */ + token?: string | undefined; + /** List what would be indexed without indexing anything. */ + dryRun?: boolean | undefined; +} + +export interface IngestContext { + db: Database.Database; + provider: EmbeddingProvider; + /** URL fetch settings (from config.indexing). */ + fetchOptions?: Pick<FetchOptions, "allowPrivateUrls" | "allowSelfSignedCerts"> | undefined; + submittedBy?: IndexDocumentInput["submittedBy"]; + /** Custom chunker for inline content and local files. */ + chunker?: Chunker | undefined; + signal?: AbortSignal | undefined; + onProgress?: + | ((p: { done: number; total?: number | undefined; message?: string | undefined }) => void) + | undefined; +} + +export interface IngestedDocument { + documentId: string; + title: string; + chunkCount: number; + /** File path or URL the document came from ("" for inline content). */ + source: string; +} + +export interface IngestResult { + kind: IngestKind; + documents: IngestedDocument[]; + errors: Array<{ source: string; error: string }>; + /** Sources that were not indexed (unsupported format, unchanged content). */ + skipped: Array<{ source: string; reason: string }>; + /** With `dryRun`: the sources that would be indexed. */ + planned?: string[] | undefined; + /** URL crawl statistics (spider only). */ + crawl?: + | { pagesCrawled: number; pagesSkipped: number; abortReason?: string | undefined } + | undefined; +} + +export function isHttpUrl(value: string): boolean { + return value.startsWith("http://") || value.startsWith("https://"); +} + +/** + * Decide how to read an ingest input: + * inline `content` -> "content"; http(s) URL -> "repo" for a GitHub/GitLab repository URL + * (unless `spider` is set), else "url"; an existing directory -> "directory"; else "file". + */ +export function detectIngestKind(input: IngestInput): IngestKind { + if (input.kind && input.kind !== "auto") return input.kind; + if (input.content !== undefined) return "content"; + const source = input.source; + if (!source) throw new ValidationError("Provide content, or a file, directory or URL to add"); + if (isHttpUrl(source)) return !input.spider && isRepoUrl(source) ? "repo" : "url"; + let isDirectory = false; + try { + isDirectory = statSync(source).isDirectory(); + } catch { + throw new ValidationError(`File or directory not found: ${source}`); + } + return isDirectory ? "directory" : "file"; +} + +/** Glob matcher for one include/exclude pattern; a pattern without "/" also matches the file name. */ +function globMatcher(pattern: string): (relPath: string) => boolean { + const re = globToRegExp(pattern); + const nameOnly = !pattern.includes("/"); + return (relPath) => re.test(relPath) || (nameOnly && re.test(basename(relPath))); +} + +/** + * Files under `dir` (recursive, sorted, hidden files and directories skipped) that match + * `include` (default: every file) and no `exclude` glob. Paths are joined to `dir`. + */ +export function listDirectoryFiles( + dir: string, + options: { include?: string[] | undefined; exclude?: string[] | undefined } = {}, +): string[] { + const include = options.include?.map(globMatcher); + const exclude = options.exclude?.map(globMatcher) ?? []; + const entries = readdirSync(dir, { recursive: true, encoding: "utf8" }); + const files: string[] = []; + for (const entry of entries) { + const rel = toPosixPath(entry); + if (rel.split("/").some((segment) => segment.startsWith("."))) continue; + if (include && !include.some((m) => m(rel))) continue; + if (exclude.some((m) => m(rel))) continue; + const full = join(dir, entry); + if (statSync(full).isFile()) files.push(full); + } + return files.sort((a, b) => a.localeCompare(b)); +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +/** Shared per-document settings derived once from the input. */ +function documentDefaults( + input: IngestInput, + ctx: IngestContext, +): Pick< + IndexDocumentInput, + "sourceType" | "library" | "version" | "topicId" | "dedup" | "expiresAt" | "submittedBy" +> { + return { + sourceType: resolveSourceType(input), + library: input.library, + version: input.version, + topicId: input.topic, + dedup: input.dedup, + expiresAt: input.expiresAt, + submittedBy: ctx.submittedBy, + }; +} + +function emptyResult(kind: IngestKind): IngestResult { + return { kind, documents: [], errors: [], skipped: [] }; +} + +function record( + ctx: IngestContext, + input: IngestInput, + result: IngestResult, + doc: IndexedDocument, + title: string, + source: string, +): void { + if (input.tags && input.tags.length > 0) addTagsToDocument(ctx.db, doc.id, input.tags); + result.documents.push({ documentId: doc.id, title, chunkCount: doc.chunkCount, source }); +} + +async function ingestContent(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const result = emptyResult("content"); + if (!input.title?.trim()) { + throw new ValidationError("A title is required when adding content directly"); + } + if (input.dryRun) return { ...result, planned: [input.title] }; + const content = input.content ?? ""; + const doc = await indexDocument(ctx.db, ctx.provider, { + ...documentDefaults(input, ctx), + title: input.title, + content, + url: input.url, + preChunked: await ctx.chunker?.({ content, title: input.title, source: input.url ?? "" }), + }); + record(ctx, input, result, doc, input.title, input.url ?? ""); + return result; +} + +function fileTitle(filePath: string): string { + return basename(filePath).replace(/\.[^.]+$/, ""); +} + +async function ingestFile(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const result = emptyResult("file"); + const path = input.source ?? ""; + if (input.dryRun) return { ...result, planned: [path] }; + const doc = await indexFile(ctx.db, ctx.provider, path, { + url: resolve(path), + title: input.title, + topic: input.topic, + library: input.library, + version: input.version, + format: input.format, + dedup: input.dedup, + sourceType: input.sourceType, + expiresAt: input.expiresAt, + chunker: ctx.chunker, + }); + record(ctx, input, result, doc, input.title ?? fileTitle(path), path); + return result; +} + +async function ingestDirectory(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const result = emptyResult("directory"); + const files = listDirectoryFiles(input.source ?? "", input); + const supported: string[] = []; + for (const file of files) { + if (input.format ?? getParserForFile(file)) supported.push(file); + else result.skipped.push({ source: file, reason: "unsupported file format" }); + } + if (input.dryRun) return { ...result, planned: supported }; + + await forEachSequential(supported, async (file, i) => { + ctx.signal?.throwIfAborted(); + try { + const doc = await indexFile(ctx.db, ctx.provider, file, { + url: resolve(file), + topic: input.topic, + library: input.library, + version: input.version, + format: input.format, + dedup: input.dedup, + sourceType: input.sourceType, + expiresAt: input.expiresAt, + chunker: ctx.chunker, + }); + record(ctx, input, result, doc, fileTitle(file), file); + } catch (err) { + result.errors.push({ source: file, error: errorMessage(err) }); + } + ctx.onProgress?.({ done: i + 1, total: supported.length, message: file }); + }); + return result; +} + +async function ingestUrl(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const url = input.source ?? input.url ?? ""; + if (input.spider) return spiderAndIndex(ctx, url, input); + const result = emptyResult("url"); + if (input.dryRun) return { ...result, planned: [url] }; + const fetched = await fetchAndConvert(url, ctx.fetchOptions); + ctx.signal?.throwIfAborted(); + const title = input.title ?? fetched.title; + const doc = await indexDocument(ctx.db, ctx.provider, { + ...documentDefaults(input, ctx), + title, + content: fetched.content, + url, + }); + record(ctx, input, result, doc, title, url); + return result; +} + +/** Build SpiderOptions from ingest input, copying only the fields that are set. */ +function spiderOptions(ctx: IngestContext, input: IngestInput): SpiderOptions { + const options: SpiderOptions = { fetchOptions: ctx.fetchOptions ?? {} }; + if (input.maxPages !== undefined) options.maxPages = input.maxPages; + if (input.maxDepth !== undefined) options.maxDepth = input.maxDepth; + if (input.sameDomain !== undefined) options.sameDomain = input.sameDomain; + if (input.pathPrefix !== undefined) options.pathPrefix = input.pathPrefix; + if (input.excludePatterns !== undefined) options.excludePatterns = input.excludePatterns; + if (ctx.signal) options.signal = ctx.signal; + return options; +} + +/** + * Crawl `url` (see spiderUrl for limits and robots.txt handling) and index every page. + * A page that fails to index is recorded in `errors`; the crawl continues. + */ +export async function spiderAndIndex( + ctx: IngestContext, + url: string, + input: IngestInput = {}, +): Promise<IngestResult> { + const result = emptyResult("url"); + if (input.dryRun) return { ...result, planned: [url] }; + const defaults = documentDefaults(input, ctx); + const gen = spiderUrl(url, spiderOptions(ctx, input)); + + // Index each crawled page as it arrives; resolves to the crawl stats when the crawl ends. + const indexPages = async (): Promise<SpiderStats> => { + const next = await gen.next(); + if (next.done) return next.value; + const page = next.value; + try { + const doc = await indexDocument(ctx.db, ctx.provider, { + ...defaults, + title: page.title, + content: page.content, + url: page.url, + }); + record(ctx, input, result, doc, page.title, page.url); + } catch (err) { + result.errors.push({ source: page.url, error: errorMessage(err) }); + } + ctx.onProgress?.({ done: result.documents.length + result.errors.length, message: page.url }); + return indexPages(); + }; + + const stats = await indexPages(); + result.crawl = { + pagesCrawled: stats.pagesCrawled, + pagesSkipped: stats.pagesSkipped, + ...(stats.abortReason ? { abortReason: stats.abortReason } : {}), + }; + return result; +} + +async function ingestRepo(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + const result = emptyResult("repo"); + const url = input.source ?? ""; + if (input.dryRun) return { ...result, planned: [url] }; + const repo = await indexRepository( + ctx.db, + ctx.provider, + { + url, + branch: input.branch, + paths: input.paths, + extensions: input.extensions?.map((e) => (e.startsWith(".") ? e : `.${e}`)), + token: input.token, + library: input.library, + version: input.version, + topicId: input.topic, + // Repository files are library docs unless the caller says otherwise. + sourceType: input.sourceType ?? "library", + signal: ctx.signal, + }, + (message) => ctx.onProgress?.({ done: result.documents.length, message }), + ); + for (const doc of repo.documents) { + record( + ctx, + input, + result, + { id: doc.documentId, chunkCount: doc.chunkCount }, + doc.title, + `${url}#${doc.path}`, + ); + } + for (const error of repo.errors) result.errors.push({ source: url, error }); + if (repo.skipped > 0) { + result.skipped.push({ source: url, reason: `${repo.skipped} unchanged file(s)` }); + } + return result; +} + +const INGESTERS: Record< + IngestKind, + (ctx: IngestContext, input: IngestInput) => Promise<IngestResult> +> = { + content: ingestContent, + file: ingestFile, + directory: ingestDirectory, + url: ingestUrl, + repo: ingestRepo, +}; + +/** Add content, a file, a directory, a URL (optionally crawled) or a repository. */ +export async function ingest(ctx: IngestContext, input: IngestInput): Promise<IngestResult> { + ctx.signal?.throwIfAborted(); + const kind = detectIngestKind(input); + return INGESTERS[kind](ctx, input); +} diff --git a/src/core/links.ts b/src/core/links.ts index e725d71c..79b2cff6 100644 --- a/src/core/links.ts +++ b/src/core/links.ts @@ -1,18 +1,21 @@ import type Database from "better-sqlite3"; import { randomUUID } from "node:crypto"; -import { ValidationError, DocumentNotFoundError } from "../errors.js"; +import { ValidationError, DocumentNotFoundError, NotFoundError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { extractMarkdownLinks, extractWikilinks } from "./link-extractor.js"; -export type LinkType = "see_also" | "prerequisite" | "supersedes" | "related" | "references"; - -const VALID_LINK_TYPES: ReadonlySet<string> = new Set<LinkType>([ +/** Every link type accepted by {@link createLink}. */ +export const LINK_TYPES = [ "see_also", "prerequisite", "supersedes", "related", "references", -]); +] as const; + +export type LinkType = (typeof LINK_TYPES)[number]; + +const VALID_LINK_TYPES: ReadonlySet<string> = new Set<string>(LINK_TYPES); export interface DocumentLink { id: string; @@ -167,7 +170,7 @@ export function deleteLink(db: Database.Database, linkId: string): void { const log = createChildLogger({ operation: "deleteLink" }); const result = db.prepare("DELETE FROM document_links WHERE id = ?").run(linkId); if (result.changes === 0) { - throw new ValidationError(`Link not found: ${linkId}`); + throw new NotFoundError(`Link not found: ${linkId}`, "LINK_NOT_FOUND"); } log.info({ linkId }, "Document link deleted"); } diff --git a/src/core/operations/admin.ts b/src/core/operations/admin.ts new file mode 100644 index 00000000..76de926b --- /dev/null +++ b/src/core/operations/admin.ts @@ -0,0 +1,232 @@ +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { describeEmbeddingIdentity, rebuildVectorTable } from "../../db/index-meta.js"; +import { createVectorTable } from "../../db/schema.js"; +import { bulkDelete, bulkMove, bulkRetag, type BulkSelector } from "../bulk.js"; +import { findDuplicates } from "../dedup.js"; +import { exportKnowledgeBase, importFromBackup } from "../export.js"; +import { getOverview } from "../overview.js"; +import { reindex } from "../reindex.js"; +import { resolveTopicId } from "../topics.js"; +import { pruneExpiredDocuments } from "../ttl.js"; +import * as s from "./schemas.js"; +import { defineOperation, type OperationContext } from "./types.js"; + +export const overviewOperation = defineOperation({ + name: "overview", + group: "admin", + summary: + "Knowledge base overview: counts, topics, installed packs, embedding model of the index, and health", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/overview" }, + handler: (ctx) => getOverview(ctx.db, ctx.provider), +}); + +export const reindexOperation = defineOperation({ + name: "reindex", + group: "admin", + summary: "Re-embed chunks with the configured embedding model (rebuild after changing models)", + input: z.object({ + documentIds: z.array(s.documentId).optional().describe("Only these documents"), + since: z.iso + .date() + .or(z.iso.datetime()) + .optional() + .describe("Only documents created on or after"), + before: z.iso + .date() + .or(z.iso.datetime()) + .optional() + .describe("Only documents created on or before"), + batchSize: z + .number() + .int() + .min(1) + .max(500) + .optional() + .describe("Chunks per embedding call (default 50)"), + rebuild: z + .boolean() + .default(false) + .describe( + "Drop and recreate the vector table for the configured model, then re-embed everything", + ), + }), + annotations: { longRunning: true, idempotent: true }, + http: { method: "POST", path: "/admin/reindex" }, + async handler(ctx, input) { + const filtered = input.documentIds ?? input.since ?? input.before; + if (input.rebuild && filtered !== undefined) { + throw new ValidationError( + "rebuild re-embeds every chunk; it cannot be combined with filters", + ); + } + let index: string | undefined; + if (input.rebuild) { + index = describeEmbeddingIdentity(await rebuildVectorTable(ctx.db, ctx.provider)); + } else { + createVectorTable(ctx.db, ctx.provider); + } + const result = await reindex(ctx.db, ctx.provider, { + documentIds: input.documentIds, + since: input.since, + before: input.before, + batchSize: input.batchSize, + signal: ctx.signal, + onProgress: (p) => ctx.onProgress?.({ done: p.completed + p.failed, total: p.total }), + }); + return { ...result, ...(index === undefined ? {} : { rebuiltIndex: index }) }; + }, +}); + +export const dedupeOperation = defineOperation({ + name: "dedupe", + group: "admin", + summary: "Find groups of duplicate or near-duplicate documents", + input: z.object({ + threshold: z + .number() + .min(0) + .max(1) + .default(0.95) + .describe("Similarity threshold for near-duplicates"), + strategy: z + .enum(["exact", "semantic", "both"]) + .default("both") + .describe("exact: same content hash; semantic: embedding similarity"), + }), + annotations: { readOnly: true, longRunning: true }, + http: { method: "GET", path: "/admin/duplicates" }, + async handler(ctx, input) { + return { items: await findDuplicates(ctx.db, ctx.provider, input) }; + }, +}); + +function assertLocalFilesAllowed(ctx: OperationContext, what: string): void { + if (ctx.surface === "api" || ctx.surface === "mcp") { + throw new ValidationError(`${what} is only available from the CLI and SDK`); + } +} + +export const backupOperation = defineOperation({ + name: "backup", + group: "admin", + summary: "Export the whole knowledge base to a JSON file", + input: z.object({ outputPath: z.string().min(1).describe("File to write") }), + annotations: { readOnly: true }, + handler(ctx, input) { + assertLocalFilesAllowed(ctx, "Backup"); + const data = exportKnowledgeBase(ctx.db, input.outputPath); + return { path: input.outputPath, counts: data.metadata.counts }; + }, +}); + +export const restoreOperation = defineOperation({ + name: "restore", + group: "admin", + summary: "Import a backup file written by backup", + input: z.object({ backupPath: z.string().min(1).describe("Backup file to read") }), + handler(ctx, input) { + assertLocalFilesAllowed(ctx, "Restore"); + const data = importFromBackup(ctx.db, input.backupPath); + return { path: input.backupPath, counts: data.metadata.counts }; + }, +}); + +export const pruneExpiredOperation = defineOperation({ + name: "prune-expired", + group: "admin", + summary: "Delete documents whose expiry time (expiresAt) has passed", + input: z.object({}), + annotations: { destructive: true, idempotent: true }, + http: { method: "POST", path: "/admin/prune-expired" }, + handler: (ctx) => pruneExpiredDocuments(ctx.db), +}); + +const selector = { + topic: s.opt(s.topic), + library: s.opt(s.library), + sourceType: s.opt(s.sourceType), + tags: s.tags.optional().describe("Documents carrying all of these tags"), + dateFrom: z.string().min(1).optional().describe("Created on or after (ISO 8601)"), + dateTo: z.string().min(1).optional().describe("Created on or before (ISO 8601)"), + dryRun: z + .boolean() + .default(false) + .describe("Report the matching documents without changing them"), +}; + +function toSelector( + ctx: OperationContext, + input: { + topic?: string | undefined; + library?: string | undefined; + sourceType?: string | undefined; + tags?: string[] | undefined; + dateFrom?: string | undefined; + dateTo?: string | undefined; + }, +): BulkSelector { + const sel: BulkSelector = {}; + if (input.topic !== undefined) sel.topicId = resolveTopicId(ctx.db, input.topic); + if (input.library !== undefined) sel.library = input.library; + if (input.sourceType !== undefined) sel.sourceType = input.sourceType; + if (input.tags !== undefined) sel.tags = input.tags; + if (input.dateFrom !== undefined) sel.dateFrom = input.dateFrom; + if (input.dateTo !== undefined) sel.dateTo = input.dateTo; + return sel; +} + +export const bulkDeleteOperation = defineOperation({ + name: "bulk-delete", + group: "admin", + summary: "Delete every document matching the filters (at most 1000 per call)", + input: z.object(selector), + annotations: { destructive: true }, + http: { method: "POST", path: "/bulk/delete" }, + handler: (ctx, input) => bulkDelete(ctx.db, toSelector(ctx, input), input.dryRun), +}); + +export const bulkRetagOperation = defineOperation({ + name: "bulk-retag", + group: "admin", + summary: "Add and/or remove tags on every document matching the filters", + input: z.object({ + ...selector, + addTags: s.tags.optional().describe("Tags to add"), + removeTags: s.tags.optional().describe("Tags to remove"), + }), + annotations: { idempotent: true }, + http: { method: "POST", path: "/bulk/retag" }, + handler: (ctx, input) => + bulkRetag(ctx.db, toSelector(ctx, input), input.addTags, input.removeTags, input.dryRun), +}); + +export const bulkMoveOperation = defineOperation({ + name: "bulk-move", + group: "admin", + summary: "Move every document matching the filters to another topic", + input: z.object({ ...selector, targetTopic: s.topic.describe("Destination topic ID or name") }), + annotations: { idempotent: true }, + http: { method: "POST", path: "/bulk/move" }, + handler: (ctx, input) => + bulkMove( + ctx.db, + toSelector(ctx, input), + resolveTopicId(ctx.db, input.targetTopic), + input.dryRun, + ), +}); + +export const adminOperations = [ + overviewOperation, + reindexOperation, + dedupeOperation, + backupOperation, + restoreOperation, + pruneExpiredOperation, + bulkDeleteOperation, + bulkRetagOperation, + bulkMoveOperation, +] as const; diff --git a/src/core/operations/analytics.ts b/src/core/operations/analytics.ts new file mode 100644 index 00000000..c2ee4f2a --- /dev/null +++ b/src/core/operations/analytics.ts @@ -0,0 +1,69 @@ +import { z } from "zod"; +import { + getKnowledgeGaps, + getPopularDocuments, + getSearchAnalytics, + getStaleDocuments, + getTopQueries, +} from "../analytics.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const days = (defaultValue: number): z.ZodDefault<z.ZodNumber> => + z + .number() + .int() + .min(1) + .max(3650) + .default(defaultValue) + .describe(`Look-back days (default ${defaultValue})`); + +export const popularOperation = defineOperation({ + name: "popular", + group: "analytics", + summary: "Documents returned most often in search results", + input: z.object({ limit: s.limit(10, 100) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/analytics/popular" }, + handler: (ctx, input) => ({ items: getPopularDocuments(ctx.db, input.limit) }), +}); + +export const staleOperation = defineOperation({ + name: "stale", + group: "analytics", + summary: "Documents that no search returned in the given number of days", + input: z.object({ days: days(90) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/analytics/stale" }, + handler: (ctx, input) => ({ items: getStaleDocuments(ctx.db, input.days) }), +}); + +export const topQueriesOperation = defineOperation({ + name: "top-queries", + group: "analytics", + summary: "Most frequent search queries", + input: z.object({ limit: s.limit(10, 100) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/analytics/top-queries" }, + handler: (ctx, input) => ({ items: getTopQueries(ctx.db, input.limit) }), +}); + +export const searchAnalyticsOperation = defineOperation({ + name: "search-analytics", + group: "analytics", + summary: "Search volume, top and zero-result queries, and knowledge gaps", + input: z.object({ days: days(30) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/analytics/searches" }, + handler: (ctx, input) => ({ + ...getSearchAnalytics(ctx.db, input.days), + knowledgeGaps: getKnowledgeGaps(ctx.db, input.days), + }), +}); + +export const analyticsOperations = [ + popularOperation, + staleOperation, + topQueriesOperation, + searchAnalyticsOperation, +] as const; diff --git a/src/core/operations/connectors.ts b/src/core/operations/connectors.ts new file mode 100644 index 00000000..c5c070d0 --- /dev/null +++ b/src/core/operations/connectors.ts @@ -0,0 +1,107 @@ +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { mapSequential } from "../../utils/async.js"; +import { listNamedConnectorConfigs } from "../../connectors/index.js"; +import { + disconnectConnection, + listConnections, + savedConnectionType, + syncConnection, + type ConnectorSyncSummary, +} from "../../connectors/registry.js"; +import { CONNECTOR_TYPES } from "../../connectors/saved-config.js"; +import { opt } from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const connectionName = z + .string() + .regex(/^[\w-]+$/, "letters, digits, '_' and '-' only") + .describe("Name of a saved connection"); + +export const listConnectionsOperation = defineOperation({ + name: "list-connections", + group: "connectors", + summary: "List saved connector connections with their schedule and last sync", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/connections" }, + handler: (ctx) => ({ items: listConnections(ctx.db) }), +}); + +type SyncOutcome = + | { name: string; type: string; status: "completed"; summary: ConnectorSyncSummary } + | { name: string; status: "failed"; error: string }; + +export const syncOperation = defineOperation({ + name: "sync", + group: "connectors", + summary: "Sync one saved connection, or all of them, using the saved settings", + input: z.object({ + name: opt(connectionName), + all: z.boolean().default(false).describe("Sync every saved connection"), + }), + annotations: { longRunning: true }, + http: { method: "POST", path: "/sync" }, + validate(_ctx, input) { + if ((input.name === undefined) === !input.all) { + throw new ValidationError("Give either a connection name or all: true"); + } + if (input.name !== undefined) savedConnectionType(input.name); + }, + async handler(ctx, input): Promise<{ items: SyncOutcome[] }> { + if (input.name !== undefined) { + const result = await syncConnection(ctx.db, ctx.provider, input.name, { signal: ctx.signal }); + return { items: [{ ...result, status: "completed" }] }; + } + // One failing connection does not stop the others; each run is recorded in connector_syncs. + const names = listNamedConnectorConfigs().map((c) => c.name); + const items = await mapSequential(names, async (name, i): Promise<SyncOutcome> => { + ctx.signal?.throwIfAborted(); + let outcome: SyncOutcome; + try { + const result = await syncConnection(ctx.db, ctx.provider, name, { signal: ctx.signal }); + outcome = { ...result, status: "completed" }; + } catch (err) { + if (ctx.signal?.aborted) throw err; + outcome = { + name, + status: "failed", + error: err instanceof Error ? err.message : String(err), + }; + } + ctx.onProgress?.({ done: i + 1, total: names.length, message: name }); + return outcome; + }); + return { items }; + }, +}); + +export const disconnectOperation = defineOperation({ + name: "disconnect", + group: "connectors", + summary: "Remove a connection's documents and its saved settings (including credentials)", + input: z.object({ + name: connectionName, + type: z + .enum(CONNECTOR_TYPES) + .optional() + .describe("Connector type; needed only when no saved connection has this name"), + keepDocuments: z + .boolean() + .default(false) + .describe("Keep the documents; only delete the saved settings"), + }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/connections/:name" }, + handler: (ctx, input) => + disconnectConnection(ctx.db, input.name, { + type: input.type, + keepDocuments: input.keepDocuments, + }), +}); + +export const connectorOperations = [ + listConnectionsOperation, + syncOperation, + disconnectOperation, +] as const; diff --git a/src/core/operations/documents.ts b/src/core/operations/documents.ts new file mode 100644 index 00000000..3b2004c5 --- /dev/null +++ b/src/core/operations/documents.ts @@ -0,0 +1,336 @@ +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { + assertDocumentExists, + countDocuments, + deleteDocument, + listDocuments, + updateDocument, + type Document, +} from "../documents.js"; +import { getDocumentView } from "../document-view.js"; +import { detectIngestKind, ingest, isHttpUrl } from "../ingest.js"; +import { rateDocument } from "../ratings.js"; +import { addTagsToDocument, getDocumentTags, removeTagsFromDocument } from "../tags.js"; +import { findTopicId, resolveTopicId } from "../topics.js"; +import { getVersionHistory, rollbackToVersion } from "../versioning.js"; +import * as s from "./schemas.js"; +import { defineOperation, type ListResult, type OperationContext } from "./types.js"; + +/** A document without its content, for listings. */ +export type DocumentSummary = Omit<Document, "content" | "id"> & { + documentId: string; + contentLength: number; +}; + +export function toDocumentSummary(doc: Document): DocumentSummary { + const { id, content, ...rest } = doc; + return { documentId: id, contentLength: content.length, ...rest }; +} + +/** Topic ID for a write: the topic must exist (by ID or name). */ +function topicForWrite(ctx: OperationContext, topic: string | undefined): string | undefined { + return topic === undefined ? undefined : resolveTopicId(ctx.db, topic); +} + +/** Topic ID for a filter: a name is resolved; an unknown value filters by itself (no match). */ +export function topicForFilter( + ctx: OperationContext, + topic: string | undefined, +): string | undefined { + return topic === undefined ? undefined : (findTopicId(ctx.db, topic) ?? topic); +} + +/** What `add` reads: `source`, else `url` when no inline content is given. */ +function addSource(input: { + source?: string | undefined; + content?: string | undefined; + url?: string | undefined; +}): string | undefined { + return input.source ?? (input.content === undefined ? input.url : undefined); +} + +export const addOperation = defineOperation({ + name: "add", + group: "documents", + summary: + "Add content to the knowledge base: inline content, a URL (optionally crawled), a GitHub/GitLab repository, or (CLI/SDK only) a local file or directory", + input: z.object({ + source: z + .string() + .min(1) + .optional() + .describe("What to add: a URL, a repository URL, or (CLI/SDK) a file or directory path"), + content: z.string().min(1).optional().describe("Inline document content (markdown)"), + title: z + .string() + .min(1) + .optional() + .describe("Title (required with content; detected for files and URLs)"), + url: s.url + .optional() + .describe("Source URL. Without content, the URL is fetched; with content, it is stored"), + kind: z + .enum(["auto", "content", "file", "directory", "url", "repo"]) + .default("auto") + .describe("How to read source (default: detect from the value)"), + topic: s.opt(s.topic), + library: s.opt(s.library), + version: s.opt(s.version), + sourceType: s.sourceType + .optional() + .describe( + "Source type (default: library if library is set, topic if topic is set, else manual)", + ), + tags: s.tags.optional().describe("Tags to add to every new document"), + expiresAt: z.iso + .datetime() + .optional() + .describe("ISO 8601 time after which the document is pruned"), + dedup: z + .enum(["skip", "warn", "force"]) + .optional() + .describe( + "Duplicate handling: skip returns the existing document, warn indexes anyway, force skips the check", + ), + format: z.string().min(1).optional().describe("Force a file format, e.g. .pdf"), + include: z.array(z.string()).optional().describe("Directory: globs of files to include"), + exclude: z.array(z.string()).optional().describe("Directory: globs of files to skip"), + spider: z.boolean().default(false).describe("URL: also crawl linked pages"), + maxPages: z + .number() + .int() + .positive() + .optional() + .describe("Crawl: page limit (default 25, max 200)"), + maxDepth: z.number().int().min(0).optional().describe("Crawl: link depth (default 2, max 5)"), + sameDomain: z.boolean().optional().describe("Crawl: stay on the seed domain (default true)"), + pathPrefix: z.string().optional().describe("Crawl: only follow links under this path"), + excludePatterns: z.array(z.string()).optional().describe("Crawl: globs of URLs to skip"), + branch: z + .string() + .min(1) + .optional() + .describe("Repository: branch (default: from URL, else main)"), + paths: z.array(z.string()).optional().describe("Repository: only these subdirectories"), + extensions: z + .array(z.string()) + .optional() + .describe("Repository: file extensions (default .md, .mdx, .txt, .rst)"), + token: z + .string() + .min(1) + .optional() + .describe("Repository: access token for private repositories"), + dryRun: z.boolean().default(false).describe("List what would be added without adding it"), + }), + annotations: { longRunning: true }, + http: { method: "POST", path: "/documents" }, + validate(ctx, input) { + const source = addSource(input); + if (input.content === undefined && source === undefined) { + throw new ValidationError("Provide content, or a file, directory or URL to add"); + } + topicForWrite(ctx, input.topic); + // Remote callers must not read (or probe for) files on the server's disk. + if (ctx.surface !== "mcp" && ctx.surface !== "api") return; + const local = + input.kind === "file" || + input.kind === "directory" || + (input.kind === "auto" && input.content === undefined && !isHttpUrl(source ?? "")); + if (local) { + throw new ValidationError( + "Adding local files or directories is only available from the CLI and SDK", + ); + } + }, + async handler(ctx, input) { + const ingestInput = { + ...input, + source: addSource(input), + topic: topicForWrite(ctx, input.topic), + }; + const kind = detectIngestKind(ingestInput); + return ingest( + { + db: ctx.db, + provider: ctx.provider, + fetchOptions: { + allowPrivateUrls: ctx.config.indexing.allowPrivateUrls, + allowSelfSignedCerts: ctx.config.indexing.allowSelfSignedCerts, + }, + submittedBy: ctx.surface === "mcp" ? "model" : "manual", + chunker: ctx.chunker, + signal: ctx.signal, + onProgress: ctx.onProgress, + }, + { ...ingestInput, kind }, + ); + }, +}); + +export const getDocumentOperation = defineOperation({ + name: "get-document", + group: "documents", + summary: "Get a document with its tags, links and rating summary; long content can be paged", + input: z.object({ + documentId: s.documentId, + offset: z.number().int().min(0).default(0).describe("Character offset into the content"), + maxLength: z + .number() + .int() + .positive() + .optional() + .describe("Maximum characters of content to return (default: all)"), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents/:documentId" }, + handler: (ctx, input) => + getDocumentView(ctx.db, input.documentId, { offset: input.offset, maxLength: input.maxLength }), +}); + +export const listDocumentsOperation = defineOperation({ + name: "list-documents", + group: "documents", + summary: "List documents (newest first) with optional filters", + input: z.object({ + ...s.documentFilters, + limit: s.limit(50, 1000), + offset: s.offset, + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents" }, + handler(ctx, input): ListResult<DocumentSummary> { + const filters = { + library: input.library, + version: input.version, + topicId: topicForFilter(ctx, input.topic), + sourceType: input.sourceType, + tags: input.tags, + }; + return { + items: listDocuments(ctx.db, { ...filters, limit: input.limit, offset: input.offset }).map( + toDocumentSummary, + ), + total: countDocuments(ctx.db, filters), + limit: input.limit, + offset: input.offset, + }; + }, +}); + +/** Make the document's tags exactly `tags`. */ +function replaceTags(ctx: OperationContext, documentId: string, tags: string[]): void { + const wanted = new Set(tags.map((t) => t.trim().toLowerCase())); + const current = getDocumentTags(ctx.db, documentId).map((t) => t.name); + removeTagsFromDocument( + ctx.db, + documentId, + current.filter((t) => !wanted.has(t)), + ); + addTagsToDocument(ctx.db, documentId, [...wanted]); +} + +export const updateDocumentOperation = defineOperation({ + name: "update-document", + group: "documents", + summary: "Update a document's title, content, metadata or tags (content changes are re-indexed)", + input: z.object({ + documentId: s.documentId, + title: z.string().min(1).optional().describe("New title"), + content: z.string().min(1).optional().describe("New content (re-chunked and re-embedded)"), + library: s.library.nullable().optional().describe("New library (null clears it)"), + version: s.version.nullable().optional().describe("New version (null clears it)"), + url: s.url.nullable().optional().describe("New URL (null clears it)"), + topic: s.topic.nullable().optional().describe("New topic ID or name (null clears it)"), + tags: s.tags.optional().describe("Replace the document's tags with these"), + }), + annotations: { idempotent: true }, + http: { method: "PATCH", path: "/documents/:documentId" }, + async handler(ctx, input) { + const { documentId, title, content, tags, topic, ...meta } = input; + if (Object.values(input).filter((v) => v !== undefined).length === 1) { + throw new ValidationError("Nothing to update: give at least one field to change"); + } + assertDocumentExists(ctx.db, documentId); + const metadata = { + ...meta, + ...(topic === undefined + ? {} + : { topicId: topic === null ? null : resolveTopicId(ctx.db, topic) }), + }; + const hasDocChange = + title !== undefined || content !== undefined || Object.keys(metadata).length > 0; + if (hasDocChange) { + await updateDocument(ctx.db, ctx.provider, documentId, { title, content, metadata }); + } + if (tags) replaceTags(ctx, documentId, tags); + return getDocumentView(ctx.db, documentId, { maxLength: 0 }); + }, +}); + +export const deleteDocumentOperation = defineOperation({ + name: "delete-document", + group: "documents", + summary: "Delete a document with its chunks, vectors, tags, links and ratings", + input: z.object({ documentId: s.documentId }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/documents/:documentId" }, + handler(ctx, input) { + deleteDocument(ctx.db, input.documentId); + return { documentId: input.documentId, deleted: true }; + }, +}); + +export const rateDocumentOperation = defineOperation({ + name: "rate-document", + group: "documents", + summary: "Rate a document (1-5), optionally with feedback or a suggested correction", + input: z.object({ + documentId: s.documentId, + chunkId: s.chunkId.optional().describe("Rate one chunk of the document"), + rating: z.number().int().min(1).max(5).describe("1 (poor) to 5 (excellent)"), + feedback: z.string().min(1).optional().describe("What is good or wrong"), + suggestedCorrection: z + .string() + .min(1) + .optional() + .describe("Replacement text if the content is wrong"), + }), + http: { method: "POST", path: "/documents/:documentId/ratings" }, + handler: (ctx, input) => + rateDocument(ctx.db, { ...input, ratedBy: ctx.surface === "mcp" ? "model" : "user" }), +}); + +export const documentHistoryOperation = defineOperation({ + name: "document-history", + group: "documents", + summary: "List saved versions of a document, newest first", + input: z.object({ documentId: s.documentId }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents/:documentId/versions" }, + handler: (ctx, input) => ({ items: getVersionHistory(ctx.db, input.documentId) }), +}); + +export const rollbackDocumentOperation = defineOperation({ + name: "rollback-document", + group: "documents", + summary: "Restore a document to a saved version (the current state is saved first)", + input: z.object({ + documentId: s.documentId, + version: z.number().int().positive().describe("Version number from document-history"), + }), + http: { method: "POST", path: "/documents/:documentId/rollback" }, + handler: (ctx, input) => rollbackToVersion(ctx.db, ctx.provider, input.documentId, input.version), +}); + +export const documentOperations = [ + addOperation, + getDocumentOperation, + listDocumentsOperation, + updateDocumentOperation, + deleteDocumentOperation, + rateDocumentOperation, + documentHistoryOperation, + rollbackDocumentOperation, +] as const; diff --git a/src/core/operations/graph.ts b/src/core/operations/graph.ts new file mode 100644 index 00000000..5cd92af5 --- /dev/null +++ b/src/core/operations/graph.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; +import { buildKnowledgeGraph } from "../graph.js"; +import { topicForFilter } from "./documents.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +export const graphOperation = defineOperation({ + name: "graph", + group: "links", + summary: + "Knowledge graph: documents, topics and tags as nodes; links, tags and similarity as edges", + input: z.object({ + topic: s.opt(s.topic).describe("Only documents in this topic (ID or name)"), + tag: z.string().min(1).optional().describe("Only documents carrying this tag"), + threshold: z + .number() + .min(0) + .max(1) + .default(0.85) + .describe("Minimum similarity for a similarity edge"), + maxNodes: z + .number() + .int() + .min(1) + .max(5000) + .default(200) + .describe("Maximum document nodes (default 200)"), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/graph" }, + handler: (ctx, input) => + buildKnowledgeGraph(ctx.db, { + similarityThreshold: input.threshold, + maxNodes: input.maxNodes, + topicFilter: topicForFilter(ctx, input.topic), + tagFilter: input.tag, + }), +}); + +export const graphOperations = [graphOperation] as const; diff --git a/src/core/operations/index.ts b/src/core/operations/index.ts new file mode 100644 index 00000000..761c097c --- /dev/null +++ b/src/core/operations/index.ts @@ -0,0 +1,56 @@ +import { adminOperations } from "./admin.js"; +import { analyticsOperations } from "./analytics.js"; +import { connectorOperations } from "./connectors.js"; +import { documentOperations } from "./documents.js"; +import { graphOperations } from "./graph.js"; +import { linkOperations } from "./links.js"; +import { packOperations } from "./packs.js"; +import { registryOperations } from "./registries.js"; +import { savedSearchOperations } from "./searches.js"; +import { searchOperations } from "./search.js"; +import { tagOperations } from "./tags.js"; +import { taskOperations } from "./tasks.js"; +import { topicOperations } from "./topics.js"; +import type { Operation } from "./types.js"; +import { webhookOperations } from "./webhooks.js"; + +export * from "./types.js"; +export * from "./admin.js"; +export * from "./analytics.js"; +export * from "./connectors.js"; +export * from "./documents.js"; +export * from "./graph.js"; +export * from "./links.js"; +export * from "./packs.js"; +export * from "./registries.js"; +export * from "./searches.js"; +export * from "./search.js"; +export * from "./tags.js"; +export * from "./tasks.js"; +export * from "./topics.js"; +export * from "./webhooks.js"; + +/** Every operation, grouped in the order surfaces list them. */ +export const OPERATIONS: readonly Operation[] = [ + ...documentOperations, + ...searchOperations, + ...linkOperations, + ...graphOperations, + ...tagOperations, + ...topicOperations, + ...savedSearchOperations, + ...packOperations, + ...registryOperations, + ...connectorOperations, + ...adminOperations, + ...analyticsOperations, + ...webhookOperations, + ...taskOperations, +] as unknown as readonly Operation[]; + +const BY_NAME = new Map(OPERATIONS.map((op) => [op.name, op])); + +/** The operation called `name`, or undefined. */ +export function getOperation(name: string): Operation | undefined { + return BY_NAME.get(name); +} diff --git a/src/core/operations/links.ts b/src/core/operations/links.ts new file mode 100644 index 00000000..3afa4026 --- /dev/null +++ b/src/core/operations/links.ts @@ -0,0 +1,108 @@ +import { z } from "zod"; +import { assertDocumentExists } from "../documents.js"; +import { + LINK_TYPES, + createLink, + deleteLink, + getDocumentLinks, + getPrerequisiteChain, + listLinks, + type DocumentLink, + type DocumentLinkWithTitle, +} from "../links.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const linkType = z.enum(LINK_TYPES).describe(`Relationship type: ${LINK_TYPES.join(", ")}`); + +type LinkOut<L extends DocumentLink> = Omit<L, "id"> & { linkId: string }; + +function toLinkOut<L extends DocumentLink>(link: L): LinkOut<L> { + const { id, ...rest } = link; + return { linkId: id, ...rest }; +} + +export const linkDocumentsOperation = defineOperation({ + name: "link-documents", + group: "links", + summary: "Create a typed link from one document to another", + input: z.object({ + documentId: s.documentId.describe("Source document ID"), + targetDocumentId: s.documentId.describe("Target document ID"), + linkType, + label: z.string().min(1).optional().describe("Short description of the relationship"), + }), + annotations: { idempotent: true }, + http: { method: "POST", path: "/documents/:documentId/links" }, + handler: (ctx, input) => + toLinkOut( + createLink(ctx.db, input.documentId, input.targetDocumentId, input.linkType, input.label), + ), +}); + +export const unlinkDocumentsOperation = defineOperation({ + name: "unlink-documents", + group: "links", + summary: "Delete a link between documents", + input: z.object({ linkId: z.string().min(1).describe("Link ID") }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/links/:linkId" }, + handler(ctx, input) { + deleteLink(ctx.db, input.linkId); + return { linkId: input.linkId, deleted: true }; + }, +}); + +export const listLinksOperation = defineOperation({ + name: "list-links", + group: "links", + summary: "List links of one document (outgoing and incoming) or all links", + input: z.object({ + documentId: s.documentId.optional().describe("Only links from or to this document"), + linkType: s.opt(linkType), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/links" }, + handler(ctx, input) { + if (input.documentId === undefined) { + return { items: listLinks(ctx.db, input.linkType).map(toLinkOut) }; + } + assertDocumentExists(ctx.db, input.documentId); + const { outgoing, incoming } = getDocumentLinks(ctx.db, input.documentId); + const withDirection = ( + links: DocumentLinkWithTitle[], + direction: "outgoing" | "incoming", + ): Array<LinkOut<DocumentLinkWithTitle> & { direction: string }> => + links + .filter((l) => input.linkType === undefined || l.linkType === input.linkType) + .map((l) => ({ ...toLinkOut(l), direction })); + return { + items: [...withDirection(outgoing, "outgoing"), ...withDirection(incoming, "incoming")], + }; + }, +}); + +export const prerequisitesOperation = defineOperation({ + name: "prerequisites", + group: "links", + summary: "List the documents to read before this one (following prerequisite links)", + input: z.object({ documentId: s.documentId }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents/:documentId/prerequisites" }, + handler(ctx, input) { + assertDocumentExists(ctx.db, input.documentId); + return { + items: getPrerequisiteChain(ctx.db, input.documentId).map((d) => ({ + documentId: d.id, + title: d.title, + })), + }; + }, +}); + +export const linkOperations = [ + linkDocumentsOperation, + unlinkDocumentsOperation, + listLinksOperation, + prerequisitesOperation, +] as const; diff --git a/src/core/operations/packs.ts b/src/core/operations/packs.ts new file mode 100644 index 00000000..5813ead4 --- /dev/null +++ b/src/core/operations/packs.ts @@ -0,0 +1,180 @@ +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { resolveRegistryPack } from "../../registry/resolve.js"; +import { listRegistryPacks } from "../../registry/search.js"; +import { + createPack, + createPackFromSource, + installPack, + listInstalledPacks, + removePack, + type KnowledgePack, +} from "../packs.js"; +import { resolveTopicId } from "../topics.js"; +import * as s from "./schemas.js"; +import { defineOperation, type OperationContext } from "./types.js"; + +const pack = z.string().min(1).describe("Pack name"); +/** A configured pack registry (`libscope registry add`). */ +export const registryName = z.string().min(1).describe("Registry name"); + +function isPackFile(nameOrPath: string): boolean { + return nameOrPath.endsWith(".json") || nameOrPath.endsWith(".json.gz"); +} + +/** Files on the server's disk are read or written only from the CLI and the Node.js API. */ +export function assertLocalFilesAllowed(ctx: OperationContext, what: string): void { + if (ctx.surface === "api" || ctx.surface === "mcp") { + throw new ValidationError(`${what} is only available from the CLI and the Node.js API`); + } +} + +export const installPackOperation = defineOperation({ + name: "install-pack", + group: "packs", + summary: "Install a knowledge pack from a registry (name or name@version)", + description: + "Packs are looked up in the local copies of the configured registries (sync the registries to update them). The CLI and the Node.js API also accept a local .json/.json.gz file.", + input: z.object({ + pack: z + .string() + .min(1) + .describe("Pack name or name@version from a registry, or a local .json/.json.gz file"), + registry: registryName + .optional() + .describe("Look only in this registry (needed when several registries have the pack)"), + batchSize: z + .number() + .int() + .min(1) + .max(500) + .optional() + .describe("Documents per batch (default 10)"), + concurrency: z + .number() + .int() + .min(1) + .max(16) + .optional() + .describe("Batches embedded in parallel (default 4)"), + resumeFrom: z + .number() + .int() + .min(0) + .optional() + .describe("Skip the first N documents (resume a partial install)"), + }), + annotations: { longRunning: true, idempotent: true }, + http: { method: "POST", path: "/packs" }, + async handler(ctx, input) { + let file = input.pack; + if (isPackFile(input.pack)) { + assertLocalFilesAllowed(ctx, "Installing a pack from a file"); + if (input.registry !== undefined) { + throw new ValidationError("registry applies to pack names, not to pack files"); + } + } else { + file = (await resolveRegistryPack(input.pack, input.registry)).dataPath; + } + return installPack(ctx.db, ctx.provider, file, { + batchSize: input.batchSize, + concurrency: input.concurrency, + resumeFrom: input.resumeFrom, + signal: ctx.signal, + onProgress: (done, total, message) => ctx.onProgress?.({ done, total, message }), + }); + }, +}); + +export const removePackOperation = defineOperation({ + name: "remove-pack", + group: "packs", + summary: "Remove an installed pack and its documents", + input: z.object({ pack }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/packs/:pack" }, + handler(ctx, input) { + removePack(ctx.db, input.pack); + return { pack: input.pack, removed: true }; + }, +}); + +export const listPacksOperation = defineOperation({ + name: "list-packs", + group: "packs", + summary: "List installed packs, or the packs available in the configured registries", + input: z.object({ + available: z + .boolean() + .default(false) + .describe("List the packs in the configured registries instead of installed ones"), + registry: registryName.optional().describe("With available: only this registry"), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/packs" }, + handler(ctx, input) { + if (input.available) { + const { packs, warnings } = listRegistryPacks(input.registry); + return { items: packs, warnings }; + } + if (input.registry !== undefined) { + throw new ValidationError("registry applies only with available: true"); + } + return { items: listInstalledPacks(ctx.db) }; + }, +}); + +export const createPackOperation = defineOperation({ + name: "create-pack", + group: "packs", + summary: "Build a pack from indexed documents, or from local files, folders and URLs", + input: z.object({ + name: z.string().min(1).describe("Pack name"), + from: z + .array(z.string().min(1)) + .optional() + .describe("Files, folders or URLs to build from (default: documents in the database)"), + topic: s.opt(s.topic).describe("Database mode: only this topic's documents"), + version: z.string().min(1).optional().describe("Pack version (default 1.0.0)"), + description: z.string().min(1).optional().describe("Pack description"), + author: z.string().min(1).optional().describe("Pack author"), + license: z.string().min(1).optional().describe("Pack license"), + outputPath: z.string().min(1).optional().describe("Write the pack here (.json or .json.gz)"), + extensions: z.array(z.string()).optional().describe("Source mode: file extensions to include"), + exclude: z.array(z.string()).optional().describe("Source mode: globs to skip"), + recursive: z.boolean().default(true).describe("Source mode: walk subdirectories"), + }), + async handler(ctx, input): Promise<KnowledgePack> { + assertLocalFilesAllowed(ctx, "Creating a pack"); + const common = { + name: input.name, + version: input.version, + description: input.description, + author: input.author, + license: input.license, + outputPath: input.outputPath, + }; + if (input.from) { + return createPackFromSource({ + ...common, + from: input.from, + extensions: input.extensions, + exclude: input.exclude, + recursive: input.recursive, + onProgress: ({ file, index, total }) => + ctx.onProgress?.({ done: index + 1, total, message: file }), + }); + } + return createPack(ctx.db, { + ...common, + topic: input.topic === undefined ? undefined : resolveTopicId(ctx.db, input.topic), + }); + }, +}); + +export const packOperations = [ + installPackOperation, + removePackOperation, + listPacksOperation, + createPackOperation, +] as const; diff --git a/src/core/operations/registries.ts b/src/core/operations/registries.ts new file mode 100644 index 00000000..e5d5a42e --- /dev/null +++ b/src/core/operations/registries.ts @@ -0,0 +1,191 @@ +/** + * Pack registries: git repositories of packs, cloned under ~/.libscope/registries/<name>/. + * Only list-registries and search-registries are read-only and exposed over REST; the others + * change local clones and config (CLI and Node.js API). + */ +import { rmSync } from "node:fs"; +import { resolve as pathResolve } from "node:path"; +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { + addRegistry, + deriveRegistryName, + loadRegistries, + removeRegistry, + requireRegistry, +} from "../../registry/config.js"; +import { createRegistryRepo } from "../../registry/git.js"; +import { publishPack, publishPackToBranch, unpublishPack } from "../../registry/publish.js"; +import { parsePackSpecifier } from "../../registry/resolve.js"; +import { searchRegistries } from "../../registry/search.js"; +import { localPackCount, syncAllRegistries, syncRegistry } from "../../registry/sync.js"; +import { getRegistryCacheDir, type PublishResult } from "../../registry/types.js"; +import { assertLocalFilesAllowed, registryName } from "./packs.js"; +import { defineOperation } from "./types.js"; + +const message = z.string().min(1).optional().describe("Git commit message"); + +export const listRegistriesOperation = defineOperation({ + name: "list-registries", + group: "packs", + summary: "List the configured pack registries", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/registries" }, + handler() { + const items = loadRegistries().map((entry) => ({ + name: entry.name, + url: entry.url, + lastSyncedAt: entry.lastSyncedAt, + packs: localPackCount(entry.name), + })); + return { items }; + }, +}); + +export const addRegistryOperation = defineOperation({ + name: "add-registry", + group: "packs", + summary: "Add a git repository as a pack registry, and clone it", + input: z.object({ + url: z + .string() + .min(1) + .describe("Git URL: https://, ssh://, git@host:path, or file:/// for a local repository"), + name: registryName.optional().describe("Registry name (default: from the URL)"), + sync: z.boolean().default(true).describe("Clone the registry now"), + }), + async handler(_ctx, input) { + const name = input.name ?? deriveRegistryName(input.url); + const entry = addRegistry({ name, url: input.url, lastSyncedAt: null }); + return { name, url: entry.url, sync: input.sync ? await syncRegistry(entry) : null }; + }, +}); + +export const removeRegistryOperation = defineOperation({ + name: "remove-registry", + group: "packs", + summary: "Remove a pack registry and delete its local clone", + input: z.object({ name: registryName }), + annotations: { destructive: true }, + handler(_ctx, input) { + removeRegistry(input.name); + rmSync(getRegistryCacheDir(input.name), { recursive: true, force: true }); + return { name: input.name, removed: true }; + }, +}); + +export const syncRegistriesOperation = defineOperation({ + name: "sync-registries", + group: "packs", + summary: "Fetch the latest packs of one or all registries", + description: + "A registry that cannot be reached keeps its local copy (status offline); one that was never cloned reports status error.", + input: z.object({ + name: registryName.optional().describe("Sync only this registry (default: all)"), + }), + annotations: { idempotent: true }, + async handler(_ctx, input) { + const items = + input.name === undefined + ? await syncAllRegistries() + : [await syncRegistry(requireRegistry(input.name))]; + return { items }; + }, +}); + +export const searchRegistriesOperation = defineOperation({ + name: "search-registries", + group: "packs", + summary: "Search the packs in the configured registries by name, description, tags and author", + input: z.object({ + query: z.string().min(1).describe("Search text"), + registry: registryName.optional().describe("Search only this registry"), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/registries/search" }, + handler(_ctx, input) { + const { results, warnings } = searchRegistries(input.query, { registryName: input.registry }); + return { items: results, warnings }; + }, +}); + +export const createRegistryOperation = defineOperation({ + name: "create-registry", + group: "packs", + summary: "Create an empty registry git repository (push it, then add it with add-registry)", + input: z.object({ path: z.string().min(1).describe("Directory to create") }), + async handler(ctx, input) { + assertLocalFilesAllowed(ctx, "Creating a registry"); + const path = pathResolve(input.path); + await createRegistryRepo(path); + return { path }; + }, +}); + +export const publishPackOperation = defineOperation({ + name: "publish-pack", + group: "packs", + summary: "Publish a pack file to a registry (commit and push)", + input: z.object({ + file: z.string().min(1).describe("Pack file (.json or .json.gz)"), + registry: registryName, + version: z + .string() + .min(1) + .optional() + .describe("Version to publish (default: the next patch version, or the pack's version)"), + message, + submit: z + .boolean() + .default(false) + .describe( + "Push to a feature/add-<pack> branch for a pull request instead of the main branch", + ), + }), + async handler(ctx, input): Promise<PublishResult & { branch: string | null }> { + assertLocalFilesAllowed(ctx, "Publishing a pack"); + const options = { + registryName: input.registry, + packFilePath: pathResolve(input.file), + version: input.version, + commitMessage: input.message, + }; + if (input.submit) return publishPackToBranch(options); + return { ...(await publishPack(options)), branch: null }; + }, +}); + +export const unpublishPackOperation = defineOperation({ + name: "unpublish-pack", + group: "packs", + summary: "Remove one version of a pack from a registry (commit and push)", + input: z.object({ + pack: z.string().min(1).describe("name@version to remove"), + registry: registryName, + message, + }), + annotations: { destructive: true }, + async handler(_ctx, input) { + const { name, version } = parsePackSpecifier(input.pack); + if (!version) throw new ValidationError("Give the version to unpublish as name@version"); + await unpublishPack({ + registryName: input.registry, + packName: name, + version, + commitMessage: input.message, + }); + return { registry: input.registry, pack: name, version, removed: true }; + }, +}); + +export const registryOperations = [ + listRegistriesOperation, + addRegistryOperation, + removeRegistryOperation, + syncRegistriesOperation, + searchRegistriesOperation, + createRegistryOperation, + publishPackOperation, + unpublishPackOperation, +] as const; diff --git a/src/core/operations/schemas.ts b/src/core/operations/schemas.ts new file mode 100644 index 00000000..c3548fb0 --- /dev/null +++ b/src/core/operations/schemas.ts @@ -0,0 +1,39 @@ +/** Shared input fields, so every operation uses the same names, limits and descriptions. */ +import { z } from "zod"; +import { SOURCE_TYPES } from "../indexing.js"; + +export const documentId = z.string().min(1).describe("Document ID"); +export const chunkId = z.string().min(1).describe("Chunk ID"); +export const sourceType = z.enum(SOURCE_TYPES).describe("Document source type"); +export const topic = z.string().min(1).describe("Topic ID or name"); +export const library = z.string().min(1).describe("Library name"); +export const version = z.string().min(1).describe("Library version"); +export const tags = z.array(z.string().min(1)).describe("Tag names"); +export const url = z.url({ protocol: /^https?$/ }).describe("http(s) URL"); + +/** `schema.optional()` keeping its description (zod does not copy it to the wrapper). */ +export function opt<T extends z.ZodType>(schema: T): z.ZodOptional<T> { + return schema.optional().describe(schema.description ?? ""); +} + +/** `limit` with a default and maximum. */ +export function limit(defaultValue: number, max: number): z.ZodDefault<z.ZodNumber> { + return z + .number() + .int() + .min(1) + .max(max) + .default(defaultValue) + .describe(`Maximum results (default ${defaultValue}, max ${max})`); +} + +export const offset = z.number().int().min(0).default(0).describe("Results to skip (paging)"); + +/** Filters shared by search, ask and list-documents. */ +export const documentFilters = { + topic: opt(topic), + library: opt(library), + version: opt(version), + sourceType: opt(sourceType), + tags: tags.optional().describe("Only documents carrying all of these tags"), +}; diff --git a/src/core/operations/search.ts b/src/core/operations/search.ts new file mode 100644 index 00000000..24a8550e --- /dev/null +++ b/src/core/operations/search.ts @@ -0,0 +1,154 @@ +import type Database from "better-sqlite3"; +import { z } from "zod"; +import { ValidationError } from "../../errors.js"; +import { answer, type RagOptions } from "../rag.js"; +import { firstChunkId, getRelatedChunks, searchDocuments, type SearchResult } from "../search.js"; +import { topicForFilter } from "./documents.js"; +import * as s from "./schemas.js"; +import { defineOperation, type ListResult, type OperationContext } from "./types.js"; + +const minRating = z.number().min(1).max(5).optional().describe("Minimum average rating"); + +/** Chunk to start a "related" search from: `ref` as a chunk ID, else a document's first chunk. */ +function relatedSeedChunk(db: Database.Database, ref: string): string { + if (db.prepare("SELECT 1 FROM chunks WHERE id = ?").get(ref)) return ref; + return firstChunkId(db, ref); +} + +interface SearchFilters { + version?: string | undefined; + sourceType?: string | undefined; + minRating?: number | undefined; +} + +/** Filters getRelatedChunks does not apply in SQL. */ +function matchesFilters(r: SearchResult, f: SearchFilters): boolean { + if (f.version !== undefined && r.version !== f.version) return false; + if (f.sourceType !== undefined && r.sourceType !== f.sourceType) return false; + return f.minRating === undefined || (r.avgRating ?? 0) >= f.minRating; +} + +export const searchOperation = defineOperation({ + name: "search", + group: "search", + summary: + "Search the knowledge base by meaning and keywords, or find content related to a document or chunk", + input: z.object({ + query: z.string().min(1).max(10_000).optional().describe("What to search for"), + relatedTo: z + .string() + .min(1) + .optional() + .describe("Document or chunk ID: return similar content instead of running a query"), + ...s.documentFilters, + minRating, + limit: s.limit(10, 100), + offset: s.offset, + maxChunksPerDocument: z + .number() + .int() + .min(1) + .max(50) + .optional() + .describe("At most this many chunks per document (default: no limit)"), + contextChunks: z + .number() + .int() + .min(0) + .max(2) + .default(0) + .describe("Neighbouring chunks to include before and after each result"), + diversity: z + .number() + .min(0) + .max(1) + .optional() + .describe("Query search: MMR reranking, 0 = relevance only (default), 1 = most diverse"), + }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/search" }, + async handler(ctx, input): Promise<ListResult<SearchResult>> { + if ((input.query === undefined) === (input.relatedTo === undefined)) { + throw new ValidationError("Give either query or relatedTo"); + } + const topic = topicForFilter(ctx, input.topic); + if (input.relatedTo !== undefined) { + const related = getRelatedChunks(ctx.db, { + chunkId: relatedSeedChunk(ctx.db, input.relatedTo), + limit: input.offset + input.limit, + ...(topic === undefined ? {} : { topic }), + ...(input.library === undefined ? {} : { library: input.library }), + ...(input.tags === undefined ? {} : { tags: input.tags }), + }); + const matches = related.chunks.filter((r) => matchesFilters(r, input)); + return { + items: matches.slice(input.offset, input.offset + input.limit), + total: matches.length, + limit: input.limit, + offset: input.offset, + }; + } + const { results, totalCount } = await searchDocuments(ctx.db, ctx.provider, { + query: input.query ?? "", + topic, + library: input.library, + version: input.version, + source: input.sourceType, + tags: input.tags, + minRating: input.minRating, + limit: input.limit, + offset: input.offset, + maxChunksPerDocument: input.maxChunksPerDocument, + contextChunks: input.contextChunks, + diversity: input.diversity, + }); + return { items: results, total: totalCount, limit: input.limit, offset: input.offset }; + }, +}); + +const askInput = z.object({ + question: z.string().min(1).max(10_000).describe("The question"), + ...s.documentFilters, + minRating, + topK: z.number().int().min(1).max(20).default(5).describe("Chunks to retrieve as context"), + systemPrompt: z + .string() + .min(1) + .max(10_000) + .optional() + .describe("System prompt for the LLM (default: answer from the context and cite titles)"), +}); + +/** RAG options for a parsed `ask` input (topic names resolved). Shared with streaming ask. */ +export function toRagOptions(ctx: OperationContext, input: z.output<typeof askInput>): RagOptions { + return { + question: input.question, + topK: input.topK, + topic: topicForFilter(ctx, input.topic), + library: input.library, + version: input.version, + sourceType: input.sourceType, + tags: input.tags, + minRating: input.minRating, + systemPrompt: input.systemPrompt, + }; +} + +export const askOperation = defineOperation({ + name: "ask", + group: "search", + summary: + "Answer a question from the knowledge base with an LLM, or (passthrough) return the context to answer from", + input: askInput, + annotations: { readOnly: true }, + http: { method: "POST", path: "/ask" }, + handler: (ctx: OperationContext, input) => + answer( + ctx.db, + ctx.provider, + { passthrough: ctx.isPassthrough(), llm: ctx.isPassthrough() ? null : ctx.getLlm() }, + toRagOptions(ctx, input), + ), +}); + +export const searchOperations = [searchOperation, askOperation] as const; diff --git a/src/core/operations/searches.ts b/src/core/operations/searches.ts new file mode 100644 index 00000000..fa4faa30 --- /dev/null +++ b/src/core/operations/searches.ts @@ -0,0 +1,90 @@ +import { z } from "zod"; +import { + countSavedSearches, + createSavedSearch, + deleteSavedSearch, + getSavedSearch, + listSavedSearches, + recordSavedSearchRun, +} from "../saved-searches.js"; +import { searchOperation } from "./search.js"; +import * as s from "./schemas.js"; +import { defineOperation, runOperation, type ListResult } from "./types.js"; + +const savedSearch = z.string().min(1).describe("Saved search name or ID"); + +export const saveSearchOperation = defineOperation({ + name: "save-search", + group: "searches", + summary: "Save a search query and its filters under a name", + input: z.object({ + name: z.string().min(1).describe("Unique name"), + query: z.string().min(1).describe("Search query"), + ...s.documentFilters, + minRating: z.number().min(1).max(5).optional().describe("Minimum average rating"), + limit: z.number().int().min(1).max(100).optional().describe("Maximum results"), + }), + http: { method: "POST", path: "/searches" }, + handler(ctx, input) { + // Filters are stored under the search operation's field names and run through it. + const { name, query, ...filters } = input; + return createSavedSearch( + ctx.db, + name, + query, + Object.keys(filters).length > 0 ? filters : undefined, + ); + }, +}); + +export const listSavedSearchesOperation = defineOperation({ + name: "list-saved-searches", + group: "searches", + summary: "List saved searches", + input: z.object({ limit: s.limit(50, 1000), offset: s.offset }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/searches" }, + handler: (ctx, input): ListResult<ReturnType<typeof listSavedSearches>[number]> => ({ + items: listSavedSearches(ctx.db, input.limit, input.offset), + total: countSavedSearches(ctx.db), + limit: input.limit, + offset: input.offset, + }), +}); + +export const runSavedSearchOperation = defineOperation({ + name: "run-saved-search", + group: "searches", + summary: "Run a saved search", + input: z.object({ search: savedSearch }), + annotations: { readOnly: true }, + http: { method: "POST", path: "/searches/:search/run" }, + async handler(ctx, input) { + const saved = getSavedSearch(ctx.db, input.search); + const { items } = await runOperation(searchOperation, ctx, { + ...saved.filters, + query: saved.query, + }); + return { search: recordSavedSearchRun(ctx.db, saved.id, items.length), items }; + }, +}); + +export const deleteSavedSearchOperation = defineOperation({ + name: "delete-saved-search", + group: "searches", + summary: "Delete a saved search", + input: z.object({ search: savedSearch }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/searches/:search" }, + handler(ctx, input) { + deleteSavedSearch(ctx.db, input.search); + return { search: input.search, deleted: true }; + }, +}); + +export const savedSearchOperations = [ + saveSearchOperation, + listSavedSearchesOperation, + runSavedSearchOperation, + deleteSavedSearchOperation, +] as const; diff --git a/src/core/operations/tags.ts b/src/core/operations/tags.ts new file mode 100644 index 00000000..fc70fd13 --- /dev/null +++ b/src/core/operations/tags.ts @@ -0,0 +1,78 @@ +import { z } from "zod"; +import { assertDocumentExists } from "../documents.js"; +import { + addTagsToDocument, + getDocumentTags, + listTags, + removeTagsFromDocument, + suggestTags, +} from "../tags.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const documentTags = z.object({ documentId: s.documentId, tags: s.tags.min(1) }); + +export const addTagsOperation = defineOperation({ + name: "add-tags", + group: "tags", + summary: "Add tags to a document (tags are created as needed)", + input: documentTags, + annotations: { idempotent: true }, + http: { method: "POST", path: "/documents/:documentId/tags" }, + handler(ctx, input) { + assertDocumentExists(ctx.db, input.documentId); + addTagsToDocument(ctx.db, input.documentId, input.tags); + return { + documentId: input.documentId, + tags: getDocumentTags(ctx.db, input.documentId).map((t) => t.name), + }; + }, +}); + +export const removeTagsOperation = defineOperation({ + name: "remove-tags", + group: "tags", + summary: "Remove tags from a document", + input: documentTags, + annotations: { idempotent: true }, + http: { method: "DELETE", path: "/documents/:documentId/tags" }, + handler(ctx, input) { + assertDocumentExists(ctx.db, input.documentId); + const removed = removeTagsFromDocument(ctx.db, input.documentId, input.tags); + return { + documentId: input.documentId, + removed, + tags: getDocumentTags(ctx.db, input.documentId).map((t) => t.name), + }; + }, +}); + +export const listTagsOperation = defineOperation({ + name: "list-tags", + group: "tags", + summary: "List all tags with their document counts", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/tags" }, + handler: (ctx) => ({ items: listTags(ctx.db) }), +}); + +export const suggestTagsOperation = defineOperation({ + name: "suggest-tags", + group: "tags", + summary: "Suggest tags for a document from its content", + input: z.object({ documentId: s.documentId, limit: s.limit(5, 20) }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/documents/:documentId/suggested-tags" }, + handler: (ctx, input) => ({ + documentId: input.documentId, + suggestions: suggestTags(ctx.db, input.documentId, input.limit), + }), +}); + +export const tagOperations = [ + addTagsOperation, + removeTagsOperation, + listTagsOperation, + suggestTagsOperation, +] as const; diff --git a/src/core/operations/tasks.ts b/src/core/operations/tasks.ts new file mode 100644 index 00000000..ea78b249 --- /dev/null +++ b/src/core/operations/tasks.ts @@ -0,0 +1,53 @@ +import { z } from "zod"; +import { NotFoundError } from "../../errors.js"; +import { taskRegistry, type Task } from "../tasks.js"; +import { defineOperation } from "./types.js"; + +const taskId = z.string().min(1).describe("Task ID returned when a background task started"); + +function requireTask(id: string): Task { + const task = taskRegistry.get(id); + if (!task) { + throw new NotFoundError( + `Task ${id} not found or expired (finished tasks are kept for 1 hour)`, + "TASK_NOT_FOUND", + ); + } + return task; +} + +export const getTaskOperation = defineOperation({ + name: "get-task", + group: "tasks", + summary: "Status, progress and result of a background task", + input: z.object({ taskId }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/tasks/:taskId" }, + handler: (_ctx, input) => requireTask(input.taskId), +}); + +export const cancelTaskOperation = defineOperation({ + name: "cancel-task", + group: "tasks", + summary: "Request cancellation of a pending or running background task", + input: z.object({ taskId }), + annotations: { idempotent: true }, + http: { method: "POST", path: "/tasks/:taskId/cancel" }, + handler(_ctx, input) { + const outcome = taskRegistry.cancel(input.taskId); + const task = requireTask(input.taskId); + return { taskId: input.taskId, cancelRequested: outcome === "cancelled", status: task.status }; + }, +}); + +export const listTasksOperation = defineOperation({ + name: "list-tasks", + group: "tasks", + summary: "List background tasks from the last hour, newest first", + input: z.object({}), + annotations: { readOnly: true }, + http: { method: "GET", path: "/tasks" }, + handler: () => ({ items: taskRegistry.list() }), +}); + +export const taskOperations = [getTaskOperation, cancelTaskOperation, listTasksOperation] as const; diff --git a/src/core/operations/topics.ts b/src/core/operations/topics.ts new file mode 100644 index 00000000..2f56c64a --- /dev/null +++ b/src/core/operations/topics.ts @@ -0,0 +1,65 @@ +import { z } from "zod"; +import { createTopic, deleteTopic, getTopicStats, resolveTopicId } from "../topics.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +export const listTopicsOperation = defineOperation({ + name: "list-topics", + group: "topics", + summary: "List topics with their document counts", + input: z.object({ parent: s.opt(s.topic).describe("Only direct subtopics of this topic") }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/topics" }, + handler(ctx, input) { + const parentId = input.parent === undefined ? undefined : resolveTopicId(ctx.db, input.parent); + const topics = getTopicStats(ctx.db); + return { + items: + parentId === undefined ? topics : topics.filter((topic) => topic.parentId === parentId), + }; + }, +}); + +export const createTopicOperation = defineOperation({ + name: "create-topic", + group: "topics", + summary: "Create a topic", + input: z.object({ + name: z.string().min(1).describe("Topic name"), + description: z.string().min(1).optional().describe("What the topic covers"), + parent: s.opt(s.topic).describe("Parent topic ID or name"), + }), + http: { method: "POST", path: "/topics" }, + handler: (ctx, input) => + createTopic(ctx.db, { + name: input.name, + description: input.description, + parentId: input.parent === undefined ? undefined : resolveTopicId(ctx.db, input.parent), + }), +}); + +export const deleteTopicOperation = defineOperation({ + name: "delete-topic", + group: "topics", + summary: "Delete a topic, optionally with its documents", + input: z.object({ + topic: s.topic, + deleteDocuments: z + .boolean() + .default(false) + .describe("Also delete the topic's documents (default: keep them, without a topic)"), + }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/topics/:topic" }, + handler(ctx, input) { + const topicId = resolveTopicId(ctx.db, input.topic); + deleteTopic(ctx.db, topicId, { deleteDocuments: input.deleteDocuments }); + return { topicId, deleted: true }; + }, +}); + +export const topicOperations = [ + listTopicsOperation, + createTopicOperation, + deleteTopicOperation, +] as const; diff --git a/src/core/operations/types.ts b/src/core/operations/types.ts new file mode 100644 index 00000000..279979fe --- /dev/null +++ b/src/core/operations/types.ts @@ -0,0 +1,199 @@ +/** + * The operation layer: every user-visible action is one Operation (name + zod input schema + + * handler). The CLI, MCP server, REST API and SDK are thin adapters over these, so parameter + * names, defaults, validation and docs are the same everywhere. + */ +import type Database from "better-sqlite3"; +import { z } from "zod"; +import type { LibScopeConfig } from "../../config.js"; +import { ConfigError, ValidationError } from "../../errors.js"; +import type { EmbeddingProvider } from "../../providers/embedding.js"; +import type { Chunker } from "../indexing.js"; +import { createLlmProvider, isPassthroughMode, type LlmProvider, type LlmSurface } from "../rag.js"; +import { taskRegistry, type Task } from "../tasks.js"; + +export type Surface = LlmSurface; + +export type OperationGroup = + | "documents" + | "search" + | "links" + | "tags" + | "topics" + | "searches" + | "packs" + | "connectors" + | "admin" + | "webhooks" + | "tasks" + | "analytics"; + +export interface ProgressEvent { + done: number; + total?: number | undefined; + message?: string | undefined; +} + +export interface OperationContext { + db: Database.Database; + provider: EmbeddingProvider; + config: LibScopeConfig; + surface: Surface; + signal?: AbortSignal | undefined; + onProgress?: ((p: ProgressEvent) => void) | undefined; + /** Custom chunker used by `add` for inline content and local files (SDK only). */ + chunker?: Chunker | undefined; + /** True when `ask` should return context for the caller to answer (passthrough). */ + isPassthrough(): boolean; + /** The configured LLM provider, created on first use; null when none is configured. */ + getLlm(): LlmProvider | null; +} + +export interface OperationAnnotations { + readOnly?: boolean; + destructive?: boolean; + idempotent?: boolean; + /** May take long enough that surfaces should offer to run it as a background task. */ + longRunning?: boolean; +} + +export type HttpMethod = "GET" | "POST" | "PATCH" | "PUT" | "DELETE"; + +export interface Operation<S extends z.ZodObject = z.ZodObject, O = unknown> { + /** kebab-case, unique, e.g. "get-document". */ + name: string; + group: OperationGroup; + /** One line, used by CLI help, MCP tool description and OpenAPI summary. */ + summary: string; + description?: string; + /** Input schema. Every field has .describe(); defaults live here only. */ + input: S; + annotations?: OperationAnnotations; + /** REST mapping; `:name` path segments are filled from input fields of the same name. */ + http?: { method: HttpMethod; path: string }; + /** + * Synchronous checks that need the context (e.g. the surface or saved settings). They run + * before the handler and before a background task starts, so callers get the error at once. + */ + validate?(ctx: OperationContext, input: z.output<S>): void; + handler(ctx: OperationContext, input: z.output<S>): Promise<O> | O; +} + +/** Identity function that keeps the schema and result types of an operation. */ +export function defineOperation<S extends z.ZodObject, O>(op: Operation<S, O>): Operation<S, O> { + return op; +} + +/** "field: message; field: message" for a failed parse. */ +function formatIssues(error: z.ZodError): string { + return error.issues + .map((issue) => { + const path = issue.path.join("."); + return path ? `${path}: ${issue.message}` : issue.message; + }) + .join("; "); +} + +/** Validate `rawInput` against the operation's schema. ZodError -> ValidationError. */ +export function parseOperationInput<S extends z.ZodObject>( + op: Operation<S, unknown>, + rawInput: unknown, +): z.output<S> { + const parsed = op.input.safeParse(rawInput ?? {}); + if (!parsed.success) { + throw new ValidationError(`Invalid input for ${op.name}: ${formatIssues(parsed.error)}`); + } + return parsed.data; +} + +/** Parse `rawInput`, then run the operation's `validate` hook. */ +function prepareInput<S extends z.ZodObject>( + op: Operation<S, unknown>, + ctx: OperationContext, + rawInput: unknown, +): z.output<S> { + const input = parseOperationInput(op, rawInput); + op.validate?.(ctx, input); + return input; +} + +/** Validate `rawInput` against the operation's schema and run it. ZodError -> ValidationError. */ +export async function runOperation<S extends z.ZodObject, O>( + op: Operation<S, O>, + ctx: OperationContext, + rawInput: unknown, +): Promise<O> { + const input = prepareInput(op, ctx, rawInput); + ctx.signal?.throwIfAborted(); + return op.handler(ctx, input); +} + +/** + * Validate input now (schema and `validate` hook), then run the operation as a background + * task in the shared task registry. The task's signal and progress replace any in `ctx`. Returns the task; its + * `result` is the JSON-encoded operation result. + */ +export function startOperationTask<S extends z.ZodObject, O>( + op: Operation<S, O>, + ctx: OperationContext, + rawInput: unknown, +): Task { + const input = prepareInput(op, ctx, rawInput); + const { task } = taskRegistry.run( + "operation", + async (signal, onProgress) => { + const result = await op.handler( + { + ...ctx, + signal, + onProgress: (p) => onProgress(p.done, p.total ?? 0), + }, + input, + ); + return JSON.stringify(result ?? null); + }, + op.name, + ); + return task; +} + +export interface CreateContextOptions { + db: Database.Database; + provider: EmbeddingProvider; + config: LibScopeConfig; + surface: Surface; + signal?: AbortSignal | undefined; + onProgress?: ((p: ProgressEvent) => void) | undefined; + chunker?: Chunker | undefined; + /** Use this LLM instead of the configured one (null: none). */ + llm?: LlmProvider | null | undefined; +} + +/** Build an OperationContext. The LLM is created from config on first `getLlm()` call. */ +export function createOperationContext(options: CreateContextOptions): OperationContext { + const { llm: injected, ...rest } = options; + let llm: LlmProvider | null | undefined = injected; + return { + ...rest, + isPassthrough: (): boolean => + injected === undefined && isPassthroughMode(options.config, { surface: options.surface }), + getLlm: (): LlmProvider | null => { + if (llm !== undefined) return llm; + try { + llm = createLlmProvider(options.config, { surface: options.surface }); + } catch (err) { + if (!(err instanceof ConfigError)) throw err; + llm = null; + } + return llm; + }, + }; +} + +/** Result shape of every list operation. */ +export interface ListResult<T> { + items: T[]; + total: number; + limit: number; + offset: number; +} diff --git a/src/core/operations/webhooks.ts b/src/core/operations/webhooks.ts new file mode 100644 index 00000000..3be69805 --- /dev/null +++ b/src/core/operations/webhooks.ts @@ -0,0 +1,81 @@ +import { z } from "zod"; +import { + TEST_PING_DATA, + WEBHOOK_EVENTS, + createWebhook, + deleteWebhook, + deliverWebhook, + getWebhook, + listWebhooks, + redactWebhook, +} from "../webhooks.js"; +import * as s from "./schemas.js"; +import { defineOperation } from "./types.js"; + +const webhookId = z.string().min(1).describe("Webhook ID"); + +export const createWebhookOperation = defineOperation({ + name: "create-webhook", + group: "webhooks", + summary: "Register a URL to receive a signed POST for document and search events", + input: z.object({ + url: s.url.describe("Endpoint that receives the POST requests"), + events: z.array(z.enum(WEBHOOK_EVENTS)).min(1).describe("Events to subscribe to"), + secret: z + .string() + .min(1) + .optional() + .describe("HMAC-SHA256 signing secret (requires LIBSCOPE_SECRET_KEY to store it)"), + }), + http: { method: "POST", path: "/webhooks" }, + handler: async (ctx, input) => + redactWebhook(await createWebhook(ctx.db, input.url, input.events, input.secret)), +}); + +export const listWebhooksOperation = defineOperation({ + name: "list-webhooks", + group: "webhooks", + summary: "List registered webhooks (secrets are never shown)", + input: z.object({ limit: s.limit(50, 1000), offset: s.offset }), + annotations: { readOnly: true }, + http: { method: "GET", path: "/webhooks" }, + handler: (ctx, input) => ({ + items: listWebhooks(ctx.db, input.limit, input.offset).map(redactWebhook), + }), +}); + +export const deleteWebhookOperation = defineOperation({ + name: "delete-webhook", + group: "webhooks", + summary: "Delete a webhook", + input: z.object({ webhookId }), + annotations: { destructive: true }, + http: { method: "DELETE", path: "/webhooks/:webhookId" }, + handler(ctx, input) { + deleteWebhook(ctx.db, input.webhookId); + return { webhookId: input.webhookId, deleted: true }; + }, +}); + +export const testWebhookOperation = defineOperation({ + name: "test-webhook", + group: "webhooks", + summary: "Send a test event to a webhook and report the HTTP status", + input: z.object({ webhookId }), + http: { method: "POST", path: "/webhooks/:webhookId/test" }, + async handler(ctx, input) { + const response = await deliverWebhook( + getWebhook(ctx.db, input.webhookId), + "document.created", + TEST_PING_DATA, + ); + return { webhookId: input.webhookId, status: response.status, statusText: response.statusText }; + }, +}); + +export const webhookOperations = [ + createWebhookOperation, + listWebhooksOperation, + deleteWebhookOperation, + testWebhookOperation, +] as const; diff --git a/src/core/overview.ts b/src/core/overview.ts new file mode 100644 index 00000000..352d2705 --- /dev/null +++ b/src/core/overview.ts @@ -0,0 +1,70 @@ +import type Database from "better-sqlite3"; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import { + getVectorTableDimensions, + readEmbeddingIdentity, + toEmbeddingIdentity, + type EmbeddingIndexIdentity, +} from "../db/index-meta.js"; +import { getLogger } from "../logger.js"; +import { getStats, type OverviewStats } from "./analytics.js"; +import { listInstalledPacks, type InstalledPack } from "./packs.js"; +import { getTopicStats, type TopicStats } from "./topics.js"; + +export type HealthStatus = "ok" | "error"; + +export interface Overview { + stats: OverviewStats; + /** Every topic with its document count. */ + topics: TopicStats[]; + packs: InstalledPack[]; + index: { + /** Provider/model/vector size recorded for the vector index (empty when none). */ + stored: EmbeddingIndexIdentity; + /** The configured embedding provider, when one was given. */ + configured?: EmbeddingIndexIdentity | undefined; + /** Vector size of chunk_embeddings; undefined when the table does not exist. */ + vectorTableDimensions?: number | undefined; + }; + health: { + database: HealthStatus; + /** Full-text (keyword) index. */ + fts: HealthStatus; + /** True when the vector table exists, so semantic search can run. */ + vectorSearch: boolean; + }; +} + +function check(name: string, fn: () => unknown): HealthStatus { + try { + fn(); + return "ok"; + } catch (err) { + getLogger().warn({ err }, `Health check failed: ${name}`); + return "error"; + } +} + +/** + * Everything a user or assistant needs to orient themselves, computed in one place: + * counts, topics, installed packs, which embedding model built the index, and health. + * Replaces the separate stats / health-check / list-topics / list-packs views. + */ +export function getOverview(db: Database.Database, provider?: EmbeddingProvider): Overview { + const vectorTableDimensions = getVectorTableDimensions(db); + return { + stats: getStats(db), + topics: getTopicStats(db), + packs: listInstalledPacks(db), + index: { + stored: readEmbeddingIdentity(db), + ...(provider ? { configured: toEmbeddingIdentity(provider) } : {}), + ...(vectorTableDimensions === undefined ? {} : { vectorTableDimensions }), + }, + health: { + database: check("database", () => db.prepare("SELECT 1").get()), + fts: check("fts", () => db.prepare("SELECT COUNT(*) FROM chunks_fts").get()), + vectorSearch: vectorTableDimensions !== undefined, + }, + }; +} diff --git a/src/core/packs.ts b/src/core/packs.ts index 04832dcd..213351e5 100644 --- a/src/core/packs.ts +++ b/src/core/packs.ts @@ -12,12 +12,20 @@ import { } from "node:path"; import { gzipSync, gunzipSync } from "node:zlib"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { ValidationError, FetchError } from "../errors.js"; +import { ValidationError, NotFoundError } from "../errors.js"; import { getLogger } from "../logger.js"; -import { chunkContent, chunkContentStreaming, STREAMING_THRESHOLD } from "./indexing.js"; +import { + buildEmbeddingText, + createChunkWriter, + splitIntoChunks, + type ChunkWriter, +} from "./indexing.js"; +import { deleteChunkEmbeddings } from "./documents.js"; import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; import { suggestTagsFromText } from "./tags.js"; import { fetchAndConvert } from "./url-fetcher.js"; +import { forEachSequential } from "../utils/async.js"; +import { globToRegExp, toPosixPath } from "../utils/glob.js"; export interface PackDocument { title: string; @@ -39,13 +47,6 @@ export interface KnowledgePack { }; } -export interface PackInfo { - name: string; - version: string; - description: string; - docCount: number; -} - export interface InstalledPack { name: string; version: string; @@ -62,7 +63,6 @@ export interface InstallResult { } export interface InstallOptions { - registryUrl?: string | undefined; /** Number of documents to embed and insert per batch. Default: 10. */ batchSize?: number | undefined; /** Skip the first N documents (for resuming a partial install). Default: 0. */ @@ -71,6 +71,11 @@ export interface InstallOptions { concurrency?: number | undefined; /** Called after each batch of documents is processed. */ onProgress?: ((current: number, total: number, docTitle: string) => void) | undefined; + /** + * Abort between batches. The install is then rolled back (pack row and inserted documents + * removed) and the returned promise rejects with the signal's reason. + */ + signal?: AbortSignal | undefined; } export interface CreatePackOptions { @@ -103,8 +108,6 @@ export interface CreatePackFromSourceOptions { onProgress?: ((info: { file: string; index: number; total: number }) => void) | undefined; } -const DEFAULT_REGISTRY_URL = "https://raw.githubusercontent.com/libscope/packs/main/registry.json"; - /** Gzip magic number: first two bytes of a gzip stream. */ const GZIP_MAGIC = Buffer.from([0x1f, 0x8b]); @@ -124,7 +127,7 @@ function writePackFile(filePath: string, pack: KnowledgePack): void { } /** Read a pack file, auto-detecting gzip by magic bytes or extension. */ -function readPackFile(filePath: string): string { +export function readPackFile(filePath: string): string { const raw = readFileSync(filePath); if (raw.length >= 2 && raw[0] === GZIP_MAGIC[0] && raw[1] === GZIP_MAGIC[1]) { return gunzipSync(raw).toString("utf-8"); @@ -132,31 +135,6 @@ function readPackFile(filePath: string): string { return raw.toString("utf-8"); } -/** Validate that a registry URL uses https and is not a private IP. */ -function validateRegistryUrl(url: string): void { - let parsed: URL; - try { - parsed = new URL(url); - } catch { - throw new ValidationError("Invalid registry URL"); - } - if (parsed.protocol !== "https:") { - throw new ValidationError("Registry URL must use https"); - } - const host = parsed.hostname; - if ( - host === "localhost" || - host.startsWith("127.") || - host.startsWith("10.") || - host.startsWith("192.168.") || - host === "0.0.0.0" || - host === "::1" || - host.startsWith("169.254.") - ) { - throw new ValidationError("Registry URL must not point to a private/internal address"); - } -} - /** Validate that a value is a non-empty string, throwing with the given message if not. */ function requireString(value: unknown, message: string, allowEmpty = false): void { if (typeof value !== "string") throw new ValidationError(message); @@ -218,49 +196,10 @@ function validatePack(data: unknown): KnowledgePack { return data as KnowledgePack; } -/** List available packs from a remote registry. */ -export async function listAvailablePacks(registryUrl?: string): Promise<PackInfo[]> { - const url = registryUrl ?? DEFAULT_REGISTRY_URL; - const log = getLogger(); - - validateRegistryUrl(url); - - try { - const response = await fetch(url, { signal: AbortSignal.timeout(30_000) }); - if (!response.ok) { - throw new FetchError(`Registry returned ${response.status}: ${response.statusText}`); - } - const data: unknown = await response.json(); - if (!Array.isArray(data)) { - throw new ValidationError("Registry response is not an array"); - } - - return (data as Array<Record<string, unknown>>).map((entry) => { - const name = entry["name"]; - const version = entry["version"]; - const description = entry["description"]; - const docCount = entry["docCount"] ?? entry["doc_count"]; - return { - name: typeof name === "string" ? name : "", - version: typeof version === "string" ? version : "", - description: typeof description === "string" ? description : "", - docCount: typeof docCount === "number" ? docCount : 0, - }; - }); - } catch (err) { - log.error({ err, url }, "Failed to fetch pack registry"); - if (err instanceof ValidationError) throw err; - if (err instanceof FetchError) throw err; - throw new FetchError( - `Failed to fetch pack registry: ${err instanceof Error ? err.message : String(err)}`, - ); - } -} - /** Load a pack from a local file path. */ -function loadPackFromFile(packNameOrPath: string): KnowledgePack { - const resolved = pathResolve(packNameOrPath); - if (!pathIsAbsolute(packNameOrPath) && !resolved.startsWith(process.cwd())) { +function loadPackFromFile(packPath: string): KnowledgePack { + const resolved = pathResolve(packPath); + if (!pathIsAbsolute(packPath) && !resolved.startsWith(process.cwd())) { throw new ValidationError("Pack file path must be within the current working directory"); } try { @@ -270,31 +209,7 @@ function loadPackFromFile(packNameOrPath: string): KnowledgePack { } catch (err) { if (err instanceof ValidationError) throw err; throw new ValidationError( - `Failed to read pack file "${packNameOrPath}": ${err instanceof Error ? err.message : String(err)}`, - ); - } -} - -/** Load a pack from a remote registry. */ -async function loadPackFromRegistry( - packNameOrPath: string, - registryUrl: string, -): Promise<KnowledgePack> { - validateRegistryUrl(registryUrl); - const baseUrl = registryUrl.replace(/\/[^/]+$/, ""); - const packUrl = `${baseUrl}/${packNameOrPath}.json`; - try { - const response = await fetch(packUrl, { signal: AbortSignal.timeout(30_000) }); - if (!response.ok) { - throw new FetchError(`Pack fetch returned ${response.status}: ${response.statusText}`); - } - const data: unknown = await response.json(); - return validatePack(data); - } catch (err) { - if (err instanceof ValidationError) throw err; - if (err instanceof FetchError) throw err; - throw new FetchError( - `Failed to fetch pack "${packNameOrPath}": ${err instanceof Error ? err.message : String(err)}`, + `Failed to read pack file "${packPath}": ${err instanceof Error ? err.message : String(err)}`, ); } } @@ -335,27 +250,28 @@ type DocChunkInfo = { }; type ResolvedBatch = { docInfos: DocChunkInfo[]; - allChunks: string[]; + /** Text to embed for every chunk in the batch, in docInfos/chunk order. */ + embeddingTexts: string[]; }; /** Chunk a batch's documents on demand, right before embedding. */ function resolveBatch(batchDocs: PackDocument[]): ResolvedBatch { const docInfos: DocChunkInfo[] = []; - const allChunks: string[] = []; + const embeddingTexts: string[] = []; for (const doc of batchDocs) { const contentHash = createHash("sha256").update(doc.content).digest("hex"); - const useStreaming = doc.content.length > STREAMING_THRESHOLD; - const chunks = useStreaming ? chunkContentStreaming(doc.content) : chunkContent(doc.content); + const chunks = splitIntoChunks(doc.content); docInfos.push({ doc, docId: randomUUID(), contentHash, chunks, - chunkOffset: allChunks.length, + chunkOffset: embeddingTexts.length, }); - allChunks.push(...chunks); + // Pack documents are stored with a title only (no library/version). + embeddingTexts.push(...chunks.map((c) => buildEmbeddingText(c, { title: doc.title }))); } - return { docInfos, allChunks }; + return { docInfos, embeddingTexts }; } /** Insert a single resolved batch into the database. Returns number of documents installed. */ @@ -365,13 +281,9 @@ function insertBatchIntoDb( embeddings: number[][], packName: string, insertDoc: Database.Statement, - insertChunk: Database.Statement, - insertEmbedding: Database.Statement, + writer: ChunkWriter, ): number { - const log = getLogger(); - let batchInstalled = 0; const doInsert = db.transaction(() => { - batchInstalled = 0; for (const info of batch.docInfos) { insertDoc.run( info.docId, @@ -383,41 +295,90 @@ function insertBatchIntoDb( info.contentHash, packName, ); - for (let j = 0; j < info.chunks.length; j++) { - const chunkId = randomUUID(); - const chunkText = info.chunks[j] ?? ""; - const embedding = embeddings[info.chunkOffset + j] ?? []; - insertChunk.run(chunkId, info.docId, chunkText, j); - try { - const vecBuffer = Buffer.from(new Float32Array(embedding).buffer); - insertEmbedding.run(chunkId, vecBuffer); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - if (!message.includes("no such table")) { - log.warn({ chunkId, err }, "Failed to insert vector embedding"); - } + writer.insertChunks( + info.docId, + info.chunks, + embeddings.slice(info.chunkOffset, info.chunkOffset + info.chunks.length), + ); + } + }); + doInsert(); + return batch.docInfos.length; +} + +/** + * Run `start(i)` for each batch index with at most `concurrency` in flight, calling + * `afterEach()` each time one settles. Stops scheduling and rejects when `start` rejects, + * `afterEach` throws, or `signal` is aborted (with the signal's reason). + */ +function runBatchesConcurrently( + batchCount: number, + concurrency: number, + start: (i: number) => Promise<void>, + afterEach: () => void, + signal: AbortSignal | undefined, +): Promise<void> { + return new Promise<void>((resolve, reject) => { + let activeCount = 0; + let scheduleIdx = 0; + let settled = false; + + const fail = (err: unknown): void => { + if (settled) return; + settled = true; + reject(err instanceof Error ? err : new Error(String(err))); + }; + + const onBatchSettled = (): void => { + activeCount--; + if (settled) return; + try { + signal?.throwIfAborted(); + afterEach(); + if (scheduleIdx < batchCount) { + scheduleNext(); + } else if (activeCount === 0) { + settled = true; + resolve(); } + } catch (err) { + fail(err); } - batchInstalled++; + }; + + const onBatchFailed = (err: unknown): void => { + activeCount--; + fail(err); + }; + + function scheduleNext(): void { + while (activeCount < concurrency && scheduleIdx < batchCount) { + const i = scheduleIdx++; + activeCount++; + start(i).then(onBatchSettled, onBatchFailed); + } + } + + if (batchCount === 0) { + resolve(); + return; } + scheduleNext(); }); - doInsert(); - return batchInstalled; } -/** Install a pack from a local JSON file path or registry name. */ +/** + * Install a pack from a .json or .json.gz file (gzip is detected by magic bytes). Packs in + * registries are resolved to a file with `resolveRegistryPack` first. + */ export async function installPack( db: Database.Database, provider: EmbeddingProvider, - packNameOrPath: string, + packPath: string, options?: InstallOptions, ): Promise<InstallResult> { const log = getLogger(); - - const isLocalFile = packNameOrPath.endsWith(".json") || packNameOrPath.endsWith(".json.gz"); - const pack = isLocalFile - ? loadPackFromFile(packNameOrPath) - : await loadPackFromRegistry(packNameOrPath, options?.registryUrl ?? DEFAULT_REGISTRY_URL); + const pack = loadPackFromFile(packPath); // Check if already installed const existing = db.prepare("SELECT name FROM packs WHERE name = ?").get(pack.name) as @@ -433,6 +394,7 @@ export async function installPack( const { batchSize, concurrency, resumeFrom } = validateInstallOptions(options, total); const onProgress = options?.onProgress; const docs = resumeFrom > 0 ? pack.documents.slice(resumeFrom) : pack.documents; + options?.signal?.throwIfAborted(); log.info( { pack: pack.name, docCount: total, batchSize, concurrency, resumeFrom }, @@ -449,14 +411,7 @@ export async function installPack( INSERT INTO documents (id, source_type, title, content, url, submitted_by, content_hash, pack_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `); - const insertChunk = db.prepare(` - INSERT INTO chunks (id, document_id, content, chunk_index) - VALUES (?, ?, ?, ?) - `); - const insertEmbedding = db.prepare(` - INSERT INTO chunk_embeddings (chunk_id, embedding) - VALUES (?, ?) - `); + const writer = createChunkWriter(db); type BatchData = { batchDocs: PackDocument[] }; const batches: BatchData[] = []; @@ -484,15 +439,7 @@ export async function installPack( errors += batch.docInfos.length; } else { try { - installed += insertBatchIntoDb( - db, - batch, - embeddings, - pack.name, - insertDoc, - insertChunk, - insertEmbedding, - ); + installed += insertBatchIntoDb(db, batch, embeddings, pack.name, insertDoc, writer); } catch (err) { log.warn( { err, pack: pack.name, batchIndex: i }, @@ -511,58 +458,37 @@ export async function installPack( } } - // Semaphore-based concurrent embedding - await new Promise<void>((resolve) => { - if (batches.length === 0) { - resolve(); - return; - } - - let activeCount = 0; - let scheduleIdx = 0; - - function scheduleNext(): void { - while (activeCount < concurrency && scheduleIdx < batches.length) { - const i = scheduleIdx++; - const resolved = resolveBatch(batches[i]!.batchDocs); - activeCount++; - - let embedPromise: Promise<number[][]>; - if (resolved.allChunks.length > 0) { - try { - embedPromise = provider.embedBatch(resolved.allChunks); - } catch (err) { - embedPromise = Promise.reject(err instanceof Error ? err : new Error(String(err))); - } - } else { - embedPromise = Promise.resolve([] as number[][]); - } - - embedPromise - .then((embeddings) => { - embedResults[i] = { resolved, embeddings, success: true }; - }) - .catch((err) => { - log.warn( - { err, pack: pack.name, batchIndex: i }, - "Failed to embed batch, skipping these documents", - ); - embedResults[i] = { resolved, embeddings: [], success: false }; - }) - .finally(() => { - activeCount--; - flushInserts(); - if (scheduleIdx < batches.length) { - scheduleNext(); - } else if (activeCount === 0) { - resolve(); - } - }); - } + /** Chunk and embed batch `i`; embedding failures are recorded, not thrown. */ + async function embedBatchAt(i: number): Promise<void> { + const resolved = resolveBatch(batches[i]!.batchDocs); + try { + const embeddings = + resolved.embeddingTexts.length > 0 + ? await provider.embedBatch(resolved.embeddingTexts) + : []; + embedResults[i] = { resolved, embeddings, success: true }; + } catch (err) { + log.warn( + { err, pack: pack.name, batchIndex: i }, + "Failed to embed batch, skipping these documents", + ); + embedResults[i] = { resolved, embeddings: [], success: false }; } + } - scheduleNext(); - }); + try { + await runBatchesConcurrently( + batches.length, + concurrency, + embedBatchAt, + flushInserts, + options?.signal, + ); + } catch (err) { + log.warn({ err, pack: pack.name }, "Pack install stopped, rolling back"); + removePack(db, pack.name); + throw err; + } db.prepare("UPDATE packs SET doc_count = ? WHERE name = ?").run(installed, pack.name); @@ -579,7 +505,7 @@ export function removePack(db: Database.Database, packName: string): void { | undefined; if (!existing) { - throw new ValidationError(`Pack "${packName}" is not installed`); + throw new NotFoundError(`Pack "${packName}" is not installed`, "PACK_NOT_FOUND"); } const docIds = db.prepare("SELECT id FROM documents WHERE pack_name = ?").all(packName) as Array<{ @@ -589,9 +515,7 @@ export function removePack(db: Database.Database, packName: string): void { const deleteTransaction = db.transaction(() => { for (const { id } of docIds) { try { - db.prepare( - "DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)", - ).run(id); + deleteChunkEmbeddings(db, [id]); } catch (err) { log.debug( { err, documentId: id }, @@ -688,18 +612,6 @@ export function createPack(db: Database.Database, options: CreatePackOptions): K // Create pack from filesystem / URL sources (no database required) // --------------------------------------------------------------------------- -/** Simple glob-style pattern matching (supports * and ** wildcards). */ -function matchesExcludePattern(relativePath: string, pattern: string): boolean { - // Escape regex special chars except * and ** - // prettier-ignore - const escaped = pattern - .replaceAll(/[.+^${}()|[\]\\]/g, String.raw`\$&`) - .replaceAll("**", "\0") - .replaceAll("*", "[^/]*") - .replaceAll("\0", ".*"); - return new RegExp(`^${escaped}$`).test(relativePath); -} - /** Recursively collect files from a directory. */ function collectFiles( dir: string, @@ -709,6 +621,7 @@ function collectFiles( excludePatterns: string[], ): string[] { const results: string[] = []; + const excludeRegexes = excludePatterns.map(globToRegExp); let entries: string[]; try { entries = readdirSync(dir); @@ -723,7 +636,7 @@ function collectFiles( const rel = relative(rootDir, fullPath); // Check exclude patterns - if (excludePatterns.some((p) => matchesExcludePattern(rel, p))) { + if (excludeRegexes.some((rx) => rx.test(toPosixPath(rel)))) { continue; } @@ -819,57 +732,76 @@ async function fetchUrlToPackDoc(url: string): Promise<PackDocument | null> { return { title: fetched.title, content: fetched.content.trimEnd(), source: url, tags }; } -/** Process all file sources, collecting documents and errors. */ -async function processFileSources( - allFiles: string[], - totalCount: number, - onProgress: CreatePackFromSourceOptions["onProgress"], -): Promise<{ documents: PackDocument[]; errors: Array<{ source: string; error: string }> }> { - const log = getLogger(); +interface SourceResults { + documents: PackDocument[]; + errors: Array<{ source: string; error: string }>; +} + +/** + * Load each source in order with `load`, collecting documents and errors. A source that fails + * is reported to `onError` and skipped. `indexOffset` is added to the progress index. + */ +async function processSources( + sources: string[], + progress: { + indexOffset: number; + totalCount: number; + onProgress: CreatePackFromSourceOptions["onProgress"]; + }, + load: (source: string) => Promise<PackDocument | null>, + onError: (source: string, message: string) => void, +): Promise<SourceResults> { const documents: PackDocument[] = []; const errors: Array<{ source: string; error: string }> = []; - for (let i = 0; i < allFiles.length; i++) { - const filePath = allFiles[i]!; - onProgress?.({ file: filePath, index: i, total: totalCount }); + await forEachSequential(sources, async (source, i) => { + progress.onProgress?.({ + file: source, + index: progress.indexOffset + i, + total: progress.totalCount, + }); try { - const doc = await parseFileToPackDoc(filePath); + const doc = await load(source); if (doc) documents.push(doc); } catch (err) { const msg = err instanceof Error ? err.message : String(err); - log.warn({ file: filePath, err: msg }, "Failed to parse file, skipping"); - errors.push({ source: filePath, error: msg }); + onError(source, msg); + errors.push({ source, error: msg }); } - } + }); return { documents, errors }; } +/** Process all file sources, collecting documents and errors. */ +function processFileSources( + allFiles: string[], + totalCount: number, + onProgress: CreatePackFromSourceOptions["onProgress"], +): Promise<SourceResults> { + const log = getLogger(); + return processSources( + allFiles, + { indexOffset: 0, totalCount, onProgress }, + parseFileToPackDoc, + (file, msg) => log.warn({ file, err: msg }, "Failed to parse file, skipping"), + ); +} + /** Process all URL sources, collecting documents and errors. */ -async function processUrlSources( +function processUrlSources( urls: string[], fileOffset: number, totalCount: number, onProgress: CreatePackFromSourceOptions["onProgress"], -): Promise<{ documents: PackDocument[]; errors: Array<{ source: string; error: string }> }> { +): Promise<SourceResults> { const log = getLogger(); - const documents: PackDocument[] = []; - const errors: Array<{ source: string; error: string }> = []; - - for (let i = 0; i < urls.length; i++) { - const url = urls[i]!; - onProgress?.({ file: url, index: fileOffset + i, total: totalCount }); - try { - const doc = await fetchUrlToPackDoc(url); - if (doc) documents.push(doc); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - log.warn({ url, err: msg }, "Failed to fetch URL, skipping"); - errors.push({ source: url, error: msg }); - } - } - - return { documents, errors }; + return processSources( + urls, + { indexOffset: fileOffset, totalCount, onProgress }, + fetchUrlToPackDoc, + (url, msg) => log.warn({ url, err: msg }, "Failed to fetch URL, skipping"), + ); } /** Validate that at least one document was created, throwing if none. */ diff --git a/src/core/parsers/markdown.ts b/src/core/parsers/markdown.ts index 82cf9f08..a3b28585 100644 --- a/src/core/parsers/markdown.ts +++ b/src/core/parsers/markdown.ts @@ -1,10 +1,6 @@ -import type { DocumentParser } from "./index.js"; +import { PlainTextParser } from "./text.js"; -/** Pass-through parser for Markdown files. */ -export class MarkdownParser implements DocumentParser { - readonly extensions = [".md", ".markdown", ".mdx"]; - - parse(content: Buffer): Promise<string> { - return Promise.resolve(content.toString("utf-8")); - } +/** Pass-through parser for Markdown files (same UTF-8 decoding as plain text). */ +export class MarkdownParser extends PlainTextParser { + override readonly extensions = [".md", ".markdown", ".mdx"]; } diff --git a/src/core/parsers/text.ts b/src/core/parsers/text.ts index 1721c4c2..a549e068 100644 --- a/src/core/parsers/text.ts +++ b/src/core/parsers/text.ts @@ -2,7 +2,7 @@ import type { DocumentParser } from "./index.js"; /** Pass-through parser for plain text files. */ export class PlainTextParser implements DocumentParser { - readonly extensions = [".txt"]; + readonly extensions: string[] = [".txt"]; parse(content: Buffer): Promise<string> { return Promise.resolve(content.toString("utf-8")); diff --git a/src/core/rag.ts b/src/core/rag.ts index 935f8d8c..d372ff0f 100644 --- a/src/core/rag.ts +++ b/src/core/rag.ts @@ -1,6 +1,11 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import type { LibScopeConfig } from "../config.js"; +import { + DEFAULT_OLLAMA_URL, + resolveLlmProviderName, + type LibScopeConfig, + type LlmSurface, +} from "../config.js"; import { searchDocuments, type SearchResult } from "./search.js"; import { ConfigError, FetchError } from "../errors.js"; @@ -13,6 +18,12 @@ export interface RagOptions { topK?: number | undefined; topic?: string | undefined; library?: string | undefined; + version?: string | undefined; + /** Filter by document source type. */ + sourceType?: string | undefined; + /** Only documents carrying all of these tags. */ + tags?: string[] | undefined; + minRating?: number | undefined; systemPrompt?: string | undefined; } @@ -66,38 +77,79 @@ export function extractSources(results: SearchResult[]): RagSource[] { })); } -interface LlmConfig { - provider?: "openai" | "ollama" | "anthropic" | "passthrough"; - model?: string; - ollamaUrl?: string; - openaiApiKey?: string; - anthropicApiKey?: string; +export { resolveLlmProviderName, type LlmSurface } from "../config.js"; + +type LlmConfig = NonNullable<LibScopeConfig["llm"]>; + +/** Options for resolving `llm.provider` ("auto" depends on the calling surface). */ +export interface LlmResolveOptions { + surface?: LlmSurface | undefined; } -/** Returns true if the config is set to passthrough mode (delegate synthesis to the calling LLM). */ -export function isPassthroughMode(config: LibScopeConfig): boolean { - return config.llm?.provider === "passthrough"; +/** + * Returns true if the LLM step is delegated to the caller: `llm.provider` is "passthrough", + * or it is "auto" and the surface is "mcp" (the calling assistant is the LLM). + */ +export function isPassthroughMode( + config: LibScopeConfig, + options: LlmResolveOptions = {}, +): boolean { + return resolveLlmProviderName(config, options) === "passthrough"; } -/** Create an LLM provider from config. */ -export function createLlmProvider(config: LibScopeConfig): LlmProvider { +const NO_LLM_HINT = + "No LLM provider configured. Set an OpenAI or Anthropic API key (LIBSCOPE_OPENAI_API_KEY, " + + "OPENAI_API_KEY, LIBSCOPE_ANTHROPIC_API_KEY, ANTHROPIC_API_KEY), or run " + + '"libscope config set llm.provider <openai|anthropic|ollama|passthrough>".'; + +/** + * Create an LLM provider from config. `llm.provider` "auto" (the default) is resolved with + * resolveLlmProviderName. Throws ConfigError when no LLM is available, or when the result is + * passthrough (no LLM is created; check isPassthroughMode first). + */ +export function createLlmProvider( + config: LibScopeConfig, + options: LlmResolveOptions = {}, +): LlmProvider { const llmConfig: LlmConfig | undefined = config.llm; - const providerType = llmConfig?.provider; + const providerType = resolveLlmProviderName(config, options); if (providerType === "openai") { - return createOpenAiProvider(config.embedding, llmConfig); + return createOpenAiProvider(config.openai?.apiKey, llmConfig); } if (providerType === "ollama") { return createOllamaProvider(config.embedding, llmConfig); } if (providerType === "anthropic") { - return createAnthropicProvider(llmConfig); + return createAnthropicProvider(config.anthropic?.apiKey, llmConfig); + } + if (providerType === "passthrough") { + throw new ConfigError( + 'llm.provider resolves to "passthrough": no LLM is created and the caller writes the ' + + "answer from the retrieved context.", + ); } - throw new ConfigError( - "No LLM provider configured. Set llm.provider to 'openai', 'ollama', 'anthropic', or 'passthrough' in your config, " + - "or set LIBSCOPE_LLM_PROVIDER environment variable.", - ); + throw new ConfigError(NO_LLM_HINT); +} + +/** Retrieve the top-K search results for a RAG question (default K = 5). */ +async function retrieveResults( + db: Database.Database, + embeddingProvider: EmbeddingProvider, + options: RagOptions, +): Promise<SearchResult[]> { + const { results } = await searchDocuments(db, embeddingProvider, { + query: options.question, + topic: options.topic, + library: options.library, + version: options.version, + source: options.sourceType, + tags: options.tags, + minRating: options.minRating, + limit: options.topK ?? 5, + }); + return results; } export interface PassthroughResult { @@ -114,14 +166,7 @@ export async function getContextForQuestion( embeddingProvider: EmbeddingProvider, options: RagOptions, ): Promise<PassthroughResult> { - const topK = options.topK ?? 5; - - const { results } = await searchDocuments(db, embeddingProvider, { - query: options.question, - topic: options.topic, - library: options.library, - limit: topK, - }); + const results = await retrieveResults(db, embeddingProvider, options); return { contextPrompt: buildContextPrompt(options.question, results), @@ -130,13 +175,13 @@ export async function getContextForQuestion( } function createOpenAiProvider( - embedding: LibScopeConfig["embedding"], + apiKey: string | undefined, llmConfig: LlmConfig | undefined, ): LlmProvider { - const apiKey = llmConfig?.openaiApiKey ?? embedding.openaiApiKey; if (!apiKey) { throw new ConfigError( - "OpenAI API key is required. Set llm.openaiApiKey or embedding.openaiApiKey in config.", + "OpenAI API key is required. Set LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY, " + + 'or run "libscope config set openai.apiKey <key>".', ); } @@ -212,7 +257,7 @@ function createOllamaProvider( embedding: LibScopeConfig["embedding"], llmConfig: LlmConfig | undefined, ): LlmProvider { - const baseUrl = llmConfig?.ollamaUrl ?? embedding.ollamaUrl ?? "http://localhost:11434"; + const baseUrl = llmConfig?.url ?? embedding.url ?? DEFAULT_OLLAMA_URL; const model = llmConfig?.model ?? "llama3.2"; return { @@ -274,11 +319,14 @@ function createOllamaProvider( }; } -function createAnthropicProvider(llmConfig: LlmConfig | undefined): LlmProvider { - const apiKey = llmConfig?.anthropicApiKey ?? process.env["ANTHROPIC_API_KEY"]; +function createAnthropicProvider( + apiKey: string | undefined, + llmConfig: LlmConfig | undefined, +): LlmProvider { if (!apiKey) { throw new ConfigError( - "Anthropic API key is required. Set llm.anthropicApiKey in config or ANTHROPIC_API_KEY env var.", + "Anthropic API key is required. Set LIBSCOPE_ANTHROPIC_API_KEY or ANTHROPIC_API_KEY, " + + 'or run "libscope config set anthropic.apiKey <key>".', ); } @@ -328,14 +376,7 @@ export async function* askQuestionStream( llmProvider: LlmProvider, options: RagOptions, ): AsyncGenerator<RagStreamEvent> { - const topK = options.topK ?? 5; - - const { results } = await searchDocuments(db, embeddingProvider, { - query: options.question, - topic: options.topic, - library: options.library, - limit: topK, - }); + const results = await retrieveResults(db, embeddingProvider, options); const contextPrompt = buildContextPrompt(options.question, results); const systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT; @@ -364,14 +405,7 @@ export async function askQuestion( llmProvider: LlmProvider, options: RagOptions, ): Promise<RagResult> { - const topK = options.topK ?? 5; - - const { results } = await searchDocuments(db, embeddingProvider, { - query: options.question, - topic: options.topic, - library: options.library, - limit: topK, - }); + const results = await retrieveResults(db, embeddingProvider, options); const contextPrompt = buildContextPrompt(options.question, results); const systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT; @@ -385,3 +419,34 @@ export async function askQuestion( tokensUsed, }; } + +/** How `answer` should produce its result. */ +export interface AnswerMode { + /** Return the retrieved context instead of calling an LLM (the caller is the LLM). */ + passthrough: boolean; + /** LLM used when not in passthrough mode; null when none is configured. */ + llm: LlmProvider | null; +} + +/** An LLM answer, or (passthrough) the context prompt for the caller to answer from. */ +export type AnswerResult = + | ({ mode: "answer" } & RagResult) + | ({ mode: "context" } & PassthroughResult); + +/** + * Answer a question from the knowledge base. Every surface uses this one function: + * passthrough mode returns the context prompt and sources; otherwise the LLM writes the answer. + * @throws ConfigError when not in passthrough mode and no LLM is configured. + */ +export async function answer( + db: Database.Database, + embeddingProvider: EmbeddingProvider, + mode: AnswerMode, + options: RagOptions, +): Promise<AnswerResult> { + if (mode.passthrough) { + return { mode: "context", ...(await getContextForQuestion(db, embeddingProvider, options)) }; + } + if (!mode.llm) throw new ConfigError(NO_LLM_HINT); + return { mode: "answer", ...(await askQuestion(db, embeddingProvider, mode.llm, options)) }; +} diff --git a/src/core/ratings.ts b/src/core/ratings.ts index b7a9e52e..1de0c324 100644 --- a/src/core/ratings.ts +++ b/src/core/ratings.ts @@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto"; import { z } from "zod"; import { ValidationError, DocumentNotFoundError } from "../errors.js"; import { validateRow, validateRows } from "../db/validate.js"; +import { emitEvent } from "./events.js"; const RatingSummaryRowSchema = z.object({ avg_rating: z.number().nullable(), @@ -90,6 +91,11 @@ export function rateDocument(db: Database.Database, input: RateDocumentInput): R ratedBy, ); + emitEvent(db, "document.rated", { + documentId: input.documentId, + rating: input.rating, + feedback: input.feedback, + }); return { id, documentId: input.documentId, diff --git a/src/core/reindex.ts b/src/core/reindex.ts index 08a9ee79..c6e5866f 100644 --- a/src/core/reindex.ts +++ b/src/core/reindex.ts @@ -1,8 +1,9 @@ import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { DatabaseError } from "../errors.js"; +import { ConfigError, DatabaseError } from "../errors.js"; import { getLogger } from "../logger.js"; import { createVectorTable } from "../db/schema.js"; +import { buildEmbeddingText, createChunkWriter } from "./indexing.js"; export interface ReindexOptions { /** Only reindex chunks belonging to these document IDs. */ @@ -15,6 +16,8 @@ export interface ReindexOptions { batchSize?: number | undefined; /** Called after each batch completes. */ onProgress?: ((progress: ReindexProgress) => void) | undefined; + /** Abort between batches; the returned promise rejects with the signal's reason. */ + signal?: AbortSignal | undefined; } export interface ReindexProgress { @@ -34,6 +37,10 @@ export interface ReindexResult { interface ChunkRow { id: string; content: string; + /** Owning document's metadata, used in the embedding text (see buildEmbeddingText). */ + title?: string | null; + library?: string | null; + version?: string | null; } /** @@ -63,73 +70,98 @@ export async function reindex( // Ensure the vector table exists with the correct dimensions for this provider. // Delegates to schema.createVectorTable() — single source of truth for the DDL. try { - createVectorTable(db, provider.dimensions); - } catch { - log.warn("Could not ensure vector table — continuing anyway"); + createVectorTable(db, provider); + } catch (err) { + if (err instanceof ConfigError) throw err; + log.warn({ err }, "Could not ensure vector table — continuing anyway"); } - const deleteStmt = db.prepare("DELETE FROM chunk_embeddings WHERE chunk_id = ?"); - const insertStmt = db.prepare("INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)"); + const writer = createChunkWriter(db); let completed = 0; let failed = 0; const failedChunkIds: string[] = []; + const batchContext: BatchContext = { db, provider, writer, failedChunkIds }; - for (let i = 0; i < total; i += batchSize) { - const batch = chunks.slice(i, i + batchSize); - const texts = batch.map((c) => c.content); - const ids = batch.map((c) => c.id); - - try { - const embeddings = await provider.embedBatch(texts); - - let batchFailed = 0; - let batchSucceeded = 0; - const upsert = db.transaction(() => { - for (let j = 0; j < ids.length; j++) { - const chunkId = ids[j]!; - const embedding = embeddings[j]; - if (!embedding) { - failedChunkIds.push(chunkId); - batchFailed++; - continue; - } - try { - deleteStmt.run(chunkId); - const vecBuffer = Buffer.from(new Float32Array(embedding).buffer); - insertStmt.run(chunkId, vecBuffer); - batchSucceeded++; - } catch (err) { - log.warn({ chunkId, err }, "Failed to update embedding for chunk"); - failedChunkIds.push(chunkId); - batchFailed++; - } - } - }); - - upsert(); - failed += batchFailed; - completed += batchSucceeded; - } catch (err) { - log.error({ err, batchStart: i }, "Batch embedding failed"); - for (const id of ids) { - failedChunkIds.push(id); - } - failed += ids.length; - } + // One batch at a time: each batch writes in its own transaction. + const reindexFrom = async (start: number): Promise<void> => { + if (start >= total) return; + options.signal?.throwIfAborted(); + const batch = chunks.slice(start, start + batchSize); + const counts = await reindexBatch(batch, start, batchContext); + failed += counts.failed; + completed += counts.succeeded; options.onProgress?.({ total, completed: completed, failed, - currentChunkId: ids[ids.length - 1], + currentChunkId: batch.at(-1)?.id, }); - } + return reindexFrom(start + batchSize); + }; + + await reindexFrom(0); log.info({ total, completed, failed }, "Reindex complete"); return { total, completed, failed, failedChunkIds }; } +interface BatchContext { + db: Database.Database; + provider: EmbeddingProvider; + writer: ReturnType<typeof createChunkWriter>; + /** Receives the IDs of chunks that could not be re-embedded. */ + failedChunkIds: string[]; +} + +/** Embed one batch of chunks and replace their vectors. Returns the per-batch counts. */ +async function reindexBatch( + batch: ChunkRow[], + batchStart: number, + ctx: BatchContext, +): Promise<{ succeeded: number; failed: number }> { + const log = getLogger(); + const { failedChunkIds, writer } = ctx; + const texts = batch.map((c) => buildEmbeddingText(c.content, c)); + const ids = batch.map((c) => c.id); + + try { + const embeddings = await ctx.provider.embedBatch(texts); + + let batchFailed = 0; + let batchSucceeded = 0; + const upsert = ctx.db.transaction(() => { + for (let j = 0; j < ids.length; j++) { + const chunkId = ids[j]!; + const embedding = embeddings[j]; + if (!embedding) { + failedChunkIds.push(chunkId); + batchFailed++; + continue; + } + try { + writer.replaceEmbedding(chunkId, embedding); + batchSucceeded++; + } catch (err) { + log.warn({ chunkId, err }, "Failed to update embedding for chunk"); + failedChunkIds.push(chunkId); + batchFailed++; + } + } + }); + + upsert(); + return { succeeded: batchSucceeded, failed: batchFailed }; + } catch (err) { + log.error({ err, batchStart }, "Batch embedding failed"); + for (const id of ids) { + failedChunkIds.push(id); + } + return { succeeded: 0, failed: ids.length }; + } +} + /** Build and execute the chunk query applying optional filters. */ function queryChunks(db: Database.Database, options: ReindexOptions): ChunkRow[] { const conditions: string[] = []; @@ -153,7 +185,7 @@ function queryChunks(db: Database.Database, options: ReindexOptions): ChunkRow[] const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; const sql = ` - SELECT c.id, c.content + SELECT c.id, c.content, d.title, d.library, d.version FROM chunks c JOIN documents d ON c.document_id = d.id ${where} diff --git a/src/core/repo.ts b/src/core/repo.ts index 1854cfeb..9ec8b13e 100644 --- a/src/core/repo.ts +++ b/src/core/repo.ts @@ -1,8 +1,10 @@ import type Database from "better-sqlite3"; import { createHash } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { FetchError, ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; +import { forEachSequential } from "../utils/async.js"; import { isPrivateIP } from "./url-fetcher.js"; import { indexDocument } from "./indexing.js"; import { promises as dns } from "node:dns"; @@ -15,12 +17,44 @@ export interface RepoOptions { paths?: string[] | undefined; extensions?: string[] | undefined; token?: string | undefined; + /** Library name for indexed files (default: "<owner>/<repo>"). */ + library?: string | undefined; + /** Library version for indexed files. */ + version?: string | undefined; + /** Topic ID for indexed files. */ + topicId?: string | undefined; + /** Source type for indexed files (default: "library"). */ + sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; + /** Stops indexing between files; the promise then rejects with the signal's reason. */ + signal?: AbortSignal | undefined; } export interface RepoResult { indexed: number; skipped: number; errors: string[]; + /** Documents indexed by this run. */ + documents: Array<{ documentId: string; title: string; chunkCount: number; path: string }>; +} + +/** + * True when `url` names a GitHub or GitLab repository (or a tree inside one) rather than a + * single page: `https://github.com/owner/repo[.git]`, `.../tree/<branch>[/path]`, + * `https://gitlab.com/owner/repo`, `.../-/tree/<branch>[/path]`. + */ +export function isRepoUrl(url: string): boolean { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return false; + } + const host = parsed.hostname.toLowerCase(); + if (host !== "github.com" && host !== "gitlab.com") return false; + const segments = parsed.pathname.split("/").filter(Boolean); + if (segments.length === 2) return true; + if (host === "github.com") return segments[2] === "tree" && segments.length >= 4; + return segments[2] === "-" && segments[3] === "tree"; } export interface ParsedRepoUrl { @@ -186,11 +220,12 @@ async function fetchWithRetry({ url, token, accept }: FetchWithRetryOptions): Pr const headers = buildApiHeaders(token, accept); let lastError: Error | undefined; - for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { + /** One request: the response, or undefined when it should be retried. */ + const tryFetch = async (attempt: number): Promise<Response | undefined> => { try { const response = await fetch(url, { headers, signal: AbortSignal.timeout(30_000) }); const shouldRetry = await handleRateLimit(response); - if (shouldRetry) continue; + if (shouldRetry) return undefined; checkResponseStatus(response, url); return response; @@ -200,17 +235,21 @@ async function fetchWithRetry({ url, token, accept }: FetchWithRetryOptions): Pr if (attempt < MAX_RETRIES - 1) { await sleep(RETRY_DELAY_MS * Math.pow(2, attempt)); } + return undefined; } - } + }; - throw new FetchError( - `Failed to fetch ${url} after ${MAX_RETRIES} retries: ${lastError?.message ?? "unknown error"}`, - lastError, - ); -} + const fetchFrom = async (attempt: number): Promise<Response> => { + if (attempt >= MAX_RETRIES) { + throw new FetchError( + `Failed to fetch ${url} after ${MAX_RETRIES} retries: ${lastError?.message ?? "unknown error"}`, + lastError, + ); + } + return (await tryFetch(attempt)) ?? fetchFrom(attempt + 1); + }; -function sleep(ms: number): Promise<void> { - return new Promise((resolve) => setTimeout(resolve, ms)); + return fetchFrom(0); } // ── Tree Fetching ──────────────────────────────────────────────────────────── @@ -305,21 +344,37 @@ async function fetchGitHubContents( onProgress?.(`Found ${docFiles.length} docs`); + const rawBase = `https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/${encodeURIComponent(branch)}`; + return fetchFileContents(docFiles, (path) => `${rawBase}/${path}`, token, onProgress); +} + +/** + * Download each file in order from `fileUrl(path)`, reporting progress. A file that cannot be + * fetched is logged and skipped. + */ +async function fetchFileContents( + docFiles: Array<{ path: string }>, + fileUrl: (path: string) => string, + token: string | undefined, + onProgress?: (message: string) => void, +): Promise<RepoFile[]> { + const log = getLogger(); const files: RepoFile[] = []; - for (let i = 0; i < docFiles.length; i++) { - const item = docFiles[i]!; + await forEachSequential(docFiles, async (item, i) => { onProgress?.(`Fetching [${i + 1}/${docFiles.length}] ${item.path}`); try { - const rawUrl = `https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/${encodeURIComponent(branch)}/${item.path}`; - const contentResponse = await fetchWithRetry({ url: rawUrl, token, accept: "text/plain" }); + const contentResponse = await fetchWithRetry({ + url: fileUrl(item.path), + token, + accept: "text/plain", + }); const content = await contentResponse.text(); files.push({ path: item.path, content }); } catch (err) { log.warn({ path: item.path, err }, "Failed to fetch file content, skipping"); } - } - + }); return files; } @@ -332,7 +387,6 @@ async function fetchGitLabContents( token: string | undefined, onProgress?: (message: string) => void, ): Promise<RepoFile[]> { - const log = getLogger(); const projectId = encodeURIComponent(`${owner}/${repo}`); onProgress?.("Fetching tree..."); @@ -346,22 +400,14 @@ async function fetchGitLabContents( onProgress?.(`Found ${docFiles.length} docs`); - const files: RepoFile[] = []; - for (let i = 0; i < docFiles.length; i++) { - const item = docFiles[i]!; - onProgress?.(`Fetching [${i + 1}/${docFiles.length}] ${item.path}`); - - try { - const fileUrl = `https://gitlab.com/api/v4/projects/${projectId}/repository/files/${encodeURIComponent(item.path)}/raw?ref=${encodeURIComponent(branch)}`; - const contentResponse = await fetchWithRetry({ url: fileUrl, token, accept: "text/plain" }); - const content = await contentResponse.text(); - files.push({ path: item.path, content }); - } catch (err) { - log.warn({ path: item.path, err }, "Failed to fetch file content, skipping"); - } - } - - return files; + const ref = encodeURIComponent(branch); + return fetchFileContents( + docFiles, + (path) => + `https://gitlab.com/api/v4/projects/${projectId}/repository/files/${encodeURIComponent(path)}/raw?ref=${ref}`, + token, + onProgress, + ); } // ── Orchestrator ───────────────────────────────────────────────────────────── @@ -381,15 +427,15 @@ export async function indexRepository( const log = getLogger(); const { owner, repo, branch: urlBranch } = parseRepoUrl(options.url); const branch = options.branch ?? urlBranch ?? "main"; - const library = `${owner}/${repo}`; + const library = options.library ?? `${owner}/${repo}`; log.info({ library, branch }, "Indexing repository"); const files = await fetchRepoContents(options, onProgress); - const result: RepoResult = { indexed: 0, skipped: 0, errors: [] }; + const result: RepoResult = { indexed: 0, skipped: 0, errors: [], documents: [] }; - for (let i = 0; i < files.length; i++) { - const file = files[i]!; + await forEachSequential(files, async (file, i) => { + options.signal?.throwIfAborted(); onProgress?.(`Indexing [${i + 1}/${files.length}] ${file.path}`); try { @@ -402,7 +448,7 @@ export async function indexRepository( if (existing?.content_hash === contentHash) { result.skipped++; - continue; + return; } const title = @@ -411,22 +457,30 @@ export async function indexRepository( .pop() ?.replace(/\.[^.]+$/, "") ?? file.path; - await indexDocument(db, provider, { + const doc = await indexDocument(db, provider, { title, content: file.content, - sourceType: "library", + sourceType: options.sourceType ?? "library", library, + version: options.version, + topicId: options.topicId, url: fileUrl, submittedBy: "crawler", }); result.indexed++; + result.documents.push({ + documentId: doc.id, + title, + chunkCount: doc.chunkCount, + path: file.path, + }); } catch (err) { const message = `${file.path}: ${err instanceof Error ? err.message : String(err)}`; result.errors.push(message); log.warn({ path: file.path, err }, "Failed to index file"); } - } + }); log.info({ library, ...result }, "Repository indexing complete"); return result; diff --git a/src/core/saved-searches.ts b/src/core/saved-searches.ts index 42b8ddfa..a4bc8ab5 100644 --- a/src/core/saved-searches.ts +++ b/src/core/saved-searches.ts @@ -1,16 +1,24 @@ import { randomUUID } from "node:crypto"; import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { searchDocuments } from "./search.js"; -import type { SearchOptions, SearchResult } from "./search.js"; -import { ValidationError, DocumentNotFoundError } from "../errors.js"; +import { ValidationError, NotFoundError } from "../errors.js"; import { getLogger } from "../logger.js"; +/** Stored filters: input fields of the `search` operation (same names). */ +export interface SavedSearchFilters { + topic?: string | undefined; + library?: string | undefined; + version?: string | undefined; + sourceType?: string | undefined; + tags?: string[] | undefined; + minRating?: number | undefined; + limit?: number | undefined; +} + export interface SavedSearch { id: string; name: string; query: string; - filters: Omit<SearchOptions, "query"> | null; + filters: SavedSearchFilters | null; createdAt: string; lastRunAt: string | null; resultCount: number; @@ -27,10 +35,10 @@ interface SavedSearchRow { } function rowToSavedSearch(row: SavedSearchRow): SavedSearch { - let filters: Omit<SearchOptions, "query"> | null = null; + let filters: SavedSearchFilters | null = null; if (row.filters) { try { - filters = JSON.parse(row.filters) as Omit<SearchOptions, "query">; + filters = JSON.parse(row.filters) as SavedSearchFilters; } catch { getLogger().warn({ id: row.id }, "Failed to parse saved search filters JSON; using null"); } @@ -50,7 +58,7 @@ export function createSavedSearch( db: Database.Database, name: string, query: string, - filters?: Omit<SearchOptions, "query">, + filters?: SavedSearchFilters, ): SavedSearch { const trimmedName = name.trim(); const trimmedQuery = query.trim(); @@ -96,12 +104,16 @@ export function listSavedSearches( return rows.map(rowToSavedSearch); } +export function countSavedSearches(db: Database.Database): number { + return (db.prepare("SELECT COUNT(*) AS n FROM saved_searches").get() as { n: number }).n; +} + export function getSavedSearch(db: Database.Database, id: string): SavedSearch { const row = db.prepare("SELECT * FROM saved_searches WHERE id = ? OR name = ?").get(id, id) as | SavedSearchRow | undefined; if (!row) { - throw new DocumentNotFoundError(id); + throw new NotFoundError(`Saved search not found: ${id}`, "SAVED_SEARCH_NOT_FOUND"); } return rowToSavedSearch(row); } @@ -109,28 +121,18 @@ export function getSavedSearch(db: Database.Database, id: string): SavedSearch { export function deleteSavedSearch(db: Database.Database, id: string): void { const result = db.prepare("DELETE FROM saved_searches WHERE id = ? OR name = ?").run(id, id); if (result.changes === 0) { - throw new DocumentNotFoundError(id); + throw new NotFoundError(`Saved search not found: ${id}`, "SAVED_SEARCH_NOT_FOUND"); } } -export async function runSavedSearch( +/** Record a run of saved search `id` (time and result count); returns the updated search. */ +export function recordSavedSearchRun( db: Database.Database, - provider: EmbeddingProvider, id: string, -): Promise<{ search: SavedSearch; results: SearchResult[] }> { - const search = getSavedSearch(db, id); - - const options: SearchOptions = { - query: search.query, - ...search.filters, - }; - - const { results } = await searchDocuments(db, provider, options); - + resultCount: number, +): SavedSearch { db.prepare( "UPDATE saved_searches SET last_run_at = datetime('now'), result_count = ? WHERE id = ?", - ).run(results.length, search.id); - - const updated = getSavedSearch(db, search.id); - return { search: updated, results }; + ).run(resultCount, id); + return getSavedSearch(db, id); } diff --git a/src/core/scheduler.ts b/src/core/scheduler.ts index 213c8a83..6b2f8333 100644 --- a/src/core/scheduler.ts +++ b/src/core/scheduler.ts @@ -1,30 +1,17 @@ import type Database from "better-sqlite3"; -import { readdirSync, readFileSync, existsSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; // NOTE: @types/node-cron v3 is used with node-cron v4 — no v4 types are published yet. // The schedule() and ScheduledTask.stop() APIs are compatible across versions. import cron from "node-cron"; import type { EmbeddingProvider } from "../providers/embedding.js"; import { ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; +import { listNamedConnectorConfigs } from "../connectors/index.js"; import { - loadNamedConnectorConfig, - saveNamedConnectorConfig, - startSync, - completeSync, - failSync, -} from "../connectors/index.js"; -import { syncNotion } from "../connectors/notion.js"; -import type { NotionConfig } from "../connectors/notion.js"; -import { syncSlack } from "../connectors/slack.js"; -import type { SlackConfig } from "../connectors/slack.js"; -import { syncConfluence } from "../connectors/confluence.js"; -import type { ConfluenceConfig } from "../connectors/confluence.js"; -import { syncObsidianVault } from "../connectors/obsidian.js"; -import type { ObsidianConfig } from "../connectors/obsidian.js"; -import { syncOneNote } from "../connectors/onenote.js"; -import type { OneNoteConfig } from "../connectors/onenote.js"; + isConnectorType, + recordFailedSync, + resolveConnectorType, +} from "../connectors/saved-config.js"; +import { runSavedConnectorSync } from "../connectors/registry.js"; export interface ScheduleConfig { cronExpression: string; @@ -66,7 +53,7 @@ export interface SchedulerStatus { * Reads schedule config from connector config files (~/.libscope/connectors/<name>.json). */ export class ConnectorScheduler { - private jobs = new Map<string, ScheduledJob>(); + private readonly jobs = new Map<string, ScheduledJob>(); private started = false; constructor( @@ -175,16 +162,18 @@ export class ConnectorScheduler { job.running = true; } - const syncId = startSync(this.db, connectorType, connectorName); log.info({ connector: connectorName, type: connectorType }, "Starting scheduled sync"); try { - const stats = await this.executeSync(connectorType, connectorName); - completeSync(this.db, syncId, stats); - log.info({ connector: connectorName, ...stats }, "Scheduled sync completed"); + // The connector records the run (one connector_syncs row, under connectorName). + if (!isConnectorType(connectorType)) { + const err = new ValidationError(`Unknown connector type: ${connectorType}`); + recordFailedSync(this.db, connectorType, connectorName, err); + throw err; + } + await runSavedConnectorSync(this.db, this.provider, connectorType, connectorName); + log.info({ connector: connectorName }, "Scheduled sync completed"); } catch (err) { - const message = err instanceof Error ? err.message : String(err); - failSync(this.db, syncId, message); log.error({ connector: connectorName, err }, "Scheduled sync failed"); } finally { if (job) { @@ -193,76 +182,6 @@ export class ConnectorScheduler { } } } - - private async executeSync( - connectorType: string, - connectorName: string, - ): Promise<{ added: number; updated: number; deleted: number; errored: number }> { - switch (connectorType) { - case "notion": { - const config = loadNamedConnectorConfig<NotionConfig>(connectorName); - const result = await syncNotion(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.pagesIndexed, - updated: 0, - deleted: 0, - errored: result.errors.length, - }; - } - case "slack": { - const config = loadNamedConnectorConfig<SlackConfig>(connectorName); - const result = await syncSlack(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.messagesIndexed + result.threadsIndexed, - updated: 0, - deleted: 0, - errored: result.errors.length, - }; - } - case "confluence": { - const config = loadNamedConnectorConfig<ConfluenceConfig>(connectorName); - const result = await syncConfluence(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.pagesIndexed, - updated: result.pagesUpdated, - deleted: 0, - errored: result.errors.length, - }; - } - case "obsidian": { - const config = loadNamedConnectorConfig<ObsidianConfig>(connectorName); - const result = await syncObsidianVault(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.added, - updated: result.updated, - deleted: result.deleted, - errored: result.errors.length, - }; - } - case "onenote": { - const config = loadNamedConnectorConfig<OneNoteConfig>(connectorName); - const result = await syncOneNote(this.db, this.provider, config); - config.lastSync = new Date().toISOString(); - saveNamedConnectorConfig(connectorName, config); - return { - added: result.pagesAdded, - updated: result.pagesUpdated, - deleted: 0, - errored: result.errors.length, - }; - } - default: - throw new ValidationError(`Unknown connector type: ${connectorType}`); - } - } } /** @@ -270,34 +189,16 @@ export class ConnectorScheduler { * Each connector config can have a `schedule` field with a `cronExpression`. */ export function loadScheduleEntries(): ConnectorScheduleEntry[] { - const log = getLogger(); const entries: ConnectorScheduleEntry[] = []; - - const connectorsDir = join(homedir(), ".libscope", "connectors"); - if (!existsSync(connectorsDir)) { - return entries; - } - - const files = readdirSync(connectorsDir).filter((f) => f.endsWith(".json")); - for (const file of files) { - try { - const raw = readFileSync(join(connectorsDir, file), "utf-8"); - const config = JSON.parse(raw) as Record<string, unknown>; - const schedule = config.schedule as { cronExpression?: string } | undefined; - - if (schedule?.cronExpression) { - const connectorName = file.replace(/\.json$/, ""); - const connectorType = (config.type as string | undefined) ?? connectorName; - entries.push({ - connectorType, - connectorName, - cronExpression: schedule.cronExpression, - }); - } - } catch (err) { - log.warn({ file, err }, "Failed to read connector config for scheduling"); + for (const { name, config } of listNamedConnectorConfigs()) { + const schedule = config["schedule"] as { cronExpression?: string } | undefined; + if (schedule?.cronExpression) { + entries.push({ + connectorType: resolveConnectorType(name, config), + connectorName: name, + cronExpression: schedule.cronExpression, + }); } } - return entries; } diff --git a/src/core/search.ts b/src/core/search.ts index c0570fd1..9ee5ac97 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -4,11 +4,15 @@ import { z } from "zod"; import { withCorrelationId, createChildLogger } from "../logger.js"; import { validateCountRow } from "../utils/db-validation.js"; import { validateRow, validateRows } from "../db/validate.js"; +import { REBUILD_VECTOR_INDEX_HINT, isVectorDimensionError } from "../db/index-meta.js"; import { logSearch, recordSearchQuery } from "./analytics.js"; +import { emitEvent } from "./events.js"; +import { STOPWORDS } from "./stopwords.js"; import { performance } from "node:perf_hooks"; +import { ChunkNotFoundError, DocumentNotFoundError } from "../errors.js"; /** Build SQL clause and params for AND-logic tag filtering on a document alias. */ -function buildTagFilter( +export function buildTagFilter( tags: string[] | undefined, docAlias: string, ): { clause: string; params: unknown[] } { @@ -215,6 +219,13 @@ function reciprocalRankFusion(listA: SearchResult[], listB: SearchResult[]): Sea return computeRrfScores(map); } +const ContextCurrentRowSchema = z.object({ chunk_index: z.number() }).optional(); +const ContextChunkRowSchema = z.object({ + id: z.string(), + content: z.string(), + chunk_index: z.number(), +}); + /** Fetch neighboring chunks for a given chunk within its document. */ function fetchContextChunks( db: Database.Database, @@ -222,9 +233,8 @@ function fetchContextChunks( documentId: string, contextSize: number, ): { before: ContextChunk[]; after: ContextChunk[] } { - const CurrentRowSchema = z.object({ chunk_index: z.number() }).optional(); const currentRow = validateRow( - CurrentRowSchema, + ContextCurrentRowSchema, db .prepare(`SELECT chunk_index FROM chunks WHERE id = ? AND document_id = ?`) .get(chunkId, documentId), @@ -235,9 +245,8 @@ function fetchContextChunks( const idx = currentRow.chunk_index; - const ChunkRowSchema = z.object({ id: z.string(), content: z.string(), chunk_index: z.number() }); const beforeRows = validateRows( - ChunkRowSchema, + ContextChunkRowSchema, db .prepare( `SELECT id, content, chunk_index FROM chunks @@ -249,7 +258,7 @@ function fetchContextChunks( ); const afterRows = validateRows( - ChunkRowSchema, + ContextChunkRowSchema, db .prepare( `SELECT id, content, chunk_index FROM chunks @@ -496,6 +505,11 @@ function finalizeResponse( if (options.contextChunks) { response.results = attachContext(db, response.results, options.contextChunks); } + emitEvent(db, "search.executed", { + query: options.query, + resultCount: response.totalCount, + topicId: options.topic, + }); return response; } @@ -521,7 +535,7 @@ export async function searchDocuments( const candidateLimit = Math.min((offset + limit) * overfetchFactor, maxCandidateLimit); try { - const vectorResults = vectorSearch(db, options, vecBuffer, candidateLimit, 0); + const vectorResults = vectorSearch(db, options, vecBuffer, candidateLimit); let ftsResults: SearchResult[] | null = null; let ftsTotalCount = 0; @@ -550,9 +564,17 @@ export async function searchDocuments( recordAnalytics(db, options, response, searchMethod, startTime, analyticsEnabled); return finalizeResponse(db, response, options); } catch (err) { - if (!isVectorTableError(err)) throw err; - - log.warn({ err }, "Vector table missing, falling back to keyword search"); + const dimensionMismatch = isVectorDimensionError(err); + if (!dimensionMismatch && !isVectorTableError(err)) throw err; + + if (dimensionMismatch) { + log.warn( + { err }, + `Vector index does not match the embedding model, falling back to keyword search. ${REBUILD_VECTOR_INDEX_HINT}`, + ); + } else { + log.warn({ err }, "Vector table missing, falling back to keyword search"); + } const response = keywordSearch(db, options, limit, offset); response.results = postProcessResults(response.results, options); @@ -562,6 +584,102 @@ export async function searchDocuments( } } +// --------------------------------------------------------------------------- +// Shared SQL fragments, row schemas and row -> SearchResult mapping +// --------------------------------------------------------------------------- + +/** Document columns selected alongside every chunk search hit (alias `d`). */ +const DOC_RESULT_COLUMNS = `d.title, + d.source_type, + d.library, + d.version, + d.topic_id, + d.url`; + +/** Optional `avg_r.avg_rating` select column, present only when the rating join is used. */ +function ratingColumn(needsRatingJoin: boolean): string { + return needsRatingJoin ? ",\n avg_r.avg_rating" : ""; +} + +/** Optional LEFT JOIN of per-document average ratings (alias `avg_r`, joined to `d`). */ +function ratingJoin(needsRatingJoin: boolean): string { + return needsRatingJoin + ? ` + LEFT JOIN ( + SELECT document_id, AVG(rating) AS avg_rating + FROM ratings + GROUP BY document_id + ) avg_r ON avg_r.document_id = d.id` + : ""; +} + +const BaseResultRowSchema = z.object({ + chunk_id: z.string(), + document_id: z.string(), + chunk_content: z.string(), + title: z.string(), + source_type: z.string(), + library: z.string().nullable(), + version: z.string().nullable(), + topic_id: z.string().nullable(), + url: z.string().nullable(), +}); + +const RatedResultRowSchema = BaseResultRowSchema.extend({ + avg_rating: z.number().nullable().optional(), +}); + +const VectorRowSchema = RatedResultRowSchema.extend({ distance: z.number() }); +const KeywordRowSchema = RatedResultRowSchema; +const Fts5RowSchema = RatedResultRowSchema.extend({ fts_rank: z.number() }); +const RelatedRowSchema = BaseResultRowSchema.extend({ distance: z.number() }); + +const LinkedChunkSchema = z.object({ + id: z.string(), + document_id: z.string(), + content: z.string(), + chunk_index: z.number(), + title: z.string(), + source_type: z.string(), + library: z.string().nullable(), + version: z.string().nullable(), + topic_id: z.string().nullable(), + url: z.string().nullable(), +}); + +type ResultRow = z.infer<typeof BaseResultRowSchema> & { + avg_rating?: number | null | undefined; +}; + +/** Map a validated search row to a SearchResult with the given score and explanation. */ +function toResult(row: ResultRow, score: number, scoreExplanation: ScoreExplanation): SearchResult { + return { + documentId: row.document_id, + chunkId: row.chunk_id, + title: row.title, + content: row.chunk_content, + sourceType: row.source_type, + library: row.library, + version: row.version, + topicId: row.topic_id, + url: row.url, + score, + avgRating: row.avg_rating ?? null, + scoreExplanation, + }; +} + +/** Map a vector ANN row (with `distance`) to a SearchResult scored by similarity. */ +function vectorRowToResult(row: ResultRow & { distance: number }): SearchResult { + const similarity = 1 - row.distance; + return toResult(row, similarity, { + method: "vector" as SearchMethod, + rawScore: row.distance, + boostFactors: [], + details: `Vector similarity: distance=${row.distance.toFixed(4)}, similarity=${similarity.toFixed(4)}`, + }); +} + /** Pure vector search — returns candidates for fusion/pagination. * `limit` should already account for offset (i.e. caller passes offset+limit). */ function vectorSearch( @@ -569,11 +687,7 @@ function vectorSearch( options: SearchOptions, vecBuffer: Buffer, limit: number, - _offset: number, ): SearchResponse { - // The caller already over-fetches; use the limit directly. - const annCandidateLimit = limit; - const needsRatingJoin = options.minRating !== undefined; let sql = ` @@ -582,12 +696,7 @@ function vectorSearch( candidates.distance, c.document_id, c.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url${needsRatingJoin ? ",\n avg_r.avg_rating" : ""} + ${DOC_RESULT_COLUMNS}${ratingColumn(needsRatingJoin)} FROM ( SELECT chunk_id, distance FROM chunk_embeddings @@ -596,37 +705,16 @@ function vectorSearch( LIMIT ? ) candidates JOIN chunks c ON c.id = candidates.chunk_id - JOIN documents d ON d.id = c.document_id${ - needsRatingJoin - ? ` - LEFT JOIN ( - SELECT document_id, AVG(rating) AS avg_rating - FROM ratings - GROUP BY document_id - ) avg_r ON avg_r.document_id = d.id` - : "" - } + JOIN documents d ON d.id = c.document_id${ratingJoin(needsRatingJoin)} WHERE 1=1 `; - const params: unknown[] = [vecBuffer, annCandidateLimit]; + // The caller already over-fetches; use the limit directly as the ANN candidate limit. + const params: unknown[] = [vecBuffer, limit]; sql = appendFilters(sql, params, options, "d"); sql += ` ORDER BY candidates.distance`; - const VectorRowSchema = z.object({ - chunk_id: z.string(), - distance: z.number(), - document_id: z.string(), - chunk_content: z.string(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - avg_rating: z.number().nullable().optional(), - }); const rows = validateRows(VectorRowSchema, db.prepare(sql).all(...params), "vectorSearch.rows"); // totalCount: if we got fewer rows than the ANN candidate limit, we know @@ -637,28 +725,7 @@ function vectorSearch( return { totalCount, - results: rows.map((row) => { - const similarity = 1 - row.distance; - return { - documentId: row.document_id, - chunkId: row.chunk_id, - title: row.title, - content: row.chunk_content, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - url: row.url, - score: similarity, - avgRating: row.avg_rating ?? null, - scoreExplanation: { - method: "vector" as SearchMethod, - rawScore: row.distance, - boostFactors: [], - details: `Vector similarity: distance=${row.distance.toFixed(4)}, similarity=${similarity.toFixed(4)}`, - }, - }; - }), + results: rows.map((row) => vectorRowToResult(row)), }; } @@ -692,23 +759,9 @@ function keywordSearch( c.id AS chunk_id, c.document_id, c.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url${needsRatingJoin ? ",\n avg_r.avg_rating" : ""} + ${DOC_RESULT_COLUMNS}${ratingColumn(needsRatingJoin)} FROM chunks c - JOIN documents d ON d.id = c.document_id${ - needsRatingJoin - ? ` - LEFT JOIN ( - SELECT document_id, AVG(rating) AS avg_rating - FROM ratings - GROUP BY document_id - ) avg_r ON avg_r.document_id = d.id` - : "" - } + JOIN documents d ON d.id = c.document_id${ratingJoin(needsRatingJoin)} WHERE (${likeConditions}) `; @@ -721,18 +774,6 @@ function keywordSearch( sql += " LIMIT ? OFFSET ?"; params.push(limit, offset); - const KeywordRowSchema = z.object({ - chunk_id: z.string(), - document_id: z.string(), - chunk_content: z.string(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - avg_rating: z.number().nullable().optional(), - }); const rows = validateRows(KeywordRowSchema, db.prepare(sql).all(...params), "keywordSearch.rows"); const totalCount = lazyCount( @@ -749,25 +790,12 @@ function keywordSearch( totalCount, results: rows.map((row, index) => { const rankScore = Math.max(0, 1 - index * 0.1); - return { - documentId: row.document_id, - chunkId: row.chunk_id, - title: row.title, - content: row.chunk_content, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - url: row.url, - score: rankScore, - avgRating: row.avg_rating ?? null, - scoreExplanation: { - method: "keyword" as SearchMethod, - rawScore: rankScore, - boostFactors: [], - details: `Keyword LIKE match: rank=${index + 1}, score=${rankScore.toFixed(4)}`, - }, - }; + return toResult(row, rankScore, { + method: "keyword" as SearchMethod, + rawScore: rankScore, + boostFactors: [], + details: `Keyword LIKE match: rank=${index + 1}, score=${rankScore.toFixed(4)}`, + }); }), }; } @@ -792,12 +820,13 @@ function sanitizeFtsWord(word: string): string { return word; } +const RatingRowSchema = z.object({ document_id: z.string(), avg_rating: z.number().nullable() }); + /** Fetch avg ratings for a small set of documents and attach to results. */ function attachRatings(db: Database.Database, results: SearchResult[]): SearchResult[] { if (results.length === 0) return results; const ids = [...new Set(results.map((r) => r.documentId))]; const placeholders = ids.map(() => "?").join(", "); - const RatingRowSchema = z.object({ document_id: z.string(), avg_rating: z.number().nullable() }); const rows = validateRows( RatingRowSchema, db @@ -814,6 +843,31 @@ function attachRatings(db: Database.Database, results: SearchResult[]): SearchRe return results.map((r) => ({ ...r, avgRating: ratingMap.get(r.documentId) ?? null })); } +const SourceChunkSchema = z.object({ + id: z.string(), + document_id: z.string(), + content: z.string(), + chunk_index: z.number(), +}); +const EmbeddingRowSchema = z.object({ embedding: z.instanceof(Buffer) }); + +/** + * ID of a document's first chunk, used to find content related to a whole document. + * Throws DocumentNotFoundError for an unknown document, ChunkNotFoundError when it has no chunks. + */ +export function firstChunkId(db: Database.Database, documentId: string): string { + const row = db + .prepare( + `SELECT d.id AS document_id, c.id AS chunk_id FROM documents d + LEFT JOIN chunks c ON c.document_id = d.id + WHERE d.id = ? ORDER BY c.chunk_index LIMIT 1`, + ) + .get(documentId) as { document_id: string; chunk_id: string | null } | undefined; + if (!row) throw new DocumentNotFoundError(documentId); + if (row.chunk_id === null) throw new ChunkNotFoundError(`first chunk of ${documentId}`); + return row.chunk_id; +} + /** * Find chunks related to a given chunk by vector similarity. * Looks up the source chunk's embedding, then searches for similar chunks @@ -828,12 +882,6 @@ export function getRelatedChunks( const minScore = options.minScore ?? 0; // Look up the source chunk - const SourceChunkSchema = z.object({ - id: z.string(), - document_id: z.string(), - content: z.string(), - chunk_index: z.number(), - }); const sourceChunkRow = validateRow( SourceChunkSchema.optional(), db @@ -842,7 +890,7 @@ export function getRelatedChunks( "getRelatedChunks.sourceChunk", ); if (!sourceChunkRow) { - throw new Error(`Chunk not found: ${chunkId}`); + throw new ChunkNotFoundError(chunkId); } const sourceChunk = { @@ -855,7 +903,6 @@ export function getRelatedChunks( const excludeDocumentId = options.excludeDocumentId ?? sourceChunkRow.document_id; // Fetch the embedding for the source chunk - const EmbeddingRowSchema = z.object({ embedding: z.instanceof(Buffer) }); const embeddingRow = validateRow( EmbeddingRowSchema.optional(), db.prepare(`SELECT embedding FROM chunk_embeddings WHERE chunk_id = ?`).get(chunkId), @@ -876,12 +923,7 @@ export function getRelatedChunks( candidates.distance, c.document_id, c.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url + ${DOC_RESULT_COLUMNS} FROM ( SELECT chunk_id, distance FROM chunk_embeddings @@ -910,47 +952,13 @@ export function getRelatedChunks( sql += ` ORDER BY candidates.distance LIMIT ?`; params.push(limit * 2); // over-fetch to allow minScore filtering - const RelatedRowSchema = z.object({ - chunk_id: z.string(), - distance: z.number(), - document_id: z.string(), - chunk_content: z.string(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - }); - const rows = validateRows( RelatedRowSchema, db.prepare(sql).all(...params), "getRelatedChunks.rows", ); - let results: SearchResult[] = rows.map((row) => { - const similarity = 1 - row.distance; - return { - documentId: row.document_id, - chunkId: row.chunk_id, - title: row.title, - content: row.chunk_content, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - url: row.url, - score: similarity, - avgRating: null, - scoreExplanation: { - method: "vector" as SearchMethod, - rawScore: row.distance, - boostFactors: [], - details: `Vector similarity: distance=${row.distance.toFixed(4)}, similarity=${similarity.toFixed(4)}`, - }, - }; - }); + let results: SearchResult[] = rows.map((row) => vectorRowToResult(row)); // Apply minScore filter if (minScore > 0) { @@ -970,19 +978,6 @@ export function getRelatedChunks( linked_doc_id: string; }[]; - const LinkedChunkSchema = z.object({ - id: z.string(), - document_id: z.string(), - content: z.string(), - chunk_index: z.number(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - }); - const presentDocIds = new Set(results.map((r) => r.documentId)); for (const { linked_doc_id } of linkedDocs) { if (!presentDocIds.has(linked_doc_id)) { @@ -1002,25 +997,18 @@ export function getRelatedChunks( "getRelatedChunks.linkedChunk", ); if (linkedChunk) { - results.push({ - documentId: linkedChunk.document_id, - chunkId: linkedChunk.id, - title: linkedChunk.title, - content: linkedChunk.content, - sourceType: linkedChunk.source_type, - library: linkedChunk.library, - version: linkedChunk.version, - topicId: linkedChunk.topic_id, - url: linkedChunk.url, - score: 0.6, - avgRating: null, - scoreExplanation: { - method: "vector" as SearchMethod, - rawScore: 0.6, - boostFactors: ["linked_document"], - details: "Explicitly linked document", - }, - }); + results.push( + toResult( + { ...linkedChunk, chunk_id: linkedChunk.id, chunk_content: linkedChunk.content }, + 0.6, + { + method: "vector" as SearchMethod, + rawScore: 0.6, + boostFactors: ["linked_document"], + details: "Explicitly linked document", + }, + ), + ); } } } @@ -1033,150 +1021,97 @@ export function getRelatedChunks( return { chunks: results, sourceChunk }; } -/** FTS5-based full-text search with BM25 ranking. Uses AND logic by default. */ -function fts5Search( +/** Quote a sanitized word as an FTS5 string literal. */ +function quoteFtsWord(word: string): string { + return `"${word.replaceAll('"', '""')}"`; +} + +/** + * Run a single FTS5 MATCH query with the standard filters and pagination. + * Returns the validated rows plus the unpaginated SQL/params for lazy counting. + */ +function runFtsQuery( db: Database.Database, options: SearchOptions, + matchExpr: string, limit: number, offset: number, -): SearchResponse { - // Sanitize and escape FTS5 query: strip dangerous syntax, wrap each word in quotes. - // AND-by-default: require all terms to match for better precision. - const words = options.query - .split(/\s+/) - .map((w) => sanitizeFtsWord(w)) - .filter((w) => w.length > 0); - if (words.length === 0) return { results: [], totalCount: 0 }; - + label: string, +): { rows: Array<z.infer<typeof Fts5RowSchema>>; baseSql: string; baseParams: unknown[] } { const needsRatingJoin = options.minRating !== undefined; - - const ftsQuery = words.map((w) => `"${w.replaceAll('"', '""')}"`).join(" AND "); - const params: unknown[] = [ftsQuery]; + const params: unknown[] = [matchExpr]; let sql = ` SELECT f.chunk_id, f.document_id, f.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url, - rank AS fts_rank${needsRatingJoin ? ",\n avg_r.avg_rating" : ""} + ${DOC_RESULT_COLUMNS}, + rank AS fts_rank${ratingColumn(needsRatingJoin)} FROM chunks_fts f - JOIN documents d ON d.id = f.document_id${ - needsRatingJoin - ? ` - LEFT JOIN ( - SELECT document_id, AVG(rating) AS avg_rating - FROM ratings - GROUP BY document_id - ) avg_r ON avg_r.document_id = d.id` - : "" - } + JOIN documents d ON d.id = f.document_id${ratingJoin(needsRatingJoin)} WHERE chunks_fts MATCH ? `; sql = appendFilters(sql, params, options, "d"); - // Lazy count – may be updated if OR fallback is used - let baseSql = sql; - let baseParams = [...params]; + const baseSql = sql; + const baseParams = [...params]; sql += " ORDER BY rank LIMIT ? OFFSET ?"; params.push(limit, offset); - const Fts5RowSchema = z.object({ - chunk_id: z.string(), - document_id: z.string(), - chunk_content: z.string(), - title: z.string(), - source_type: z.string(), - library: z.string().nullable(), - version: z.string().nullable(), - topic_id: z.string().nullable(), - url: z.string().nullable(), - fts_rank: z.number(), - avg_rating: z.number().nullable().optional(), - }); - let rows = validateRows(Fts5RowSchema, db.prepare(sql).all(...params), "fts5Search.rows"); - - // If AND returned nothing, retry with OR for recall - if (rows.length === 0 && words.length > 1) { - const orQuery = words.map((w) => `"${w.replace(/"/g, '""')}"`).join(" OR "); - const orParams: unknown[] = [orQuery]; - let orSql = ` - SELECT - f.chunk_id, - f.document_id, - f.content AS chunk_content, - d.title, - d.source_type, - d.library, - d.version, - d.topic_id, - d.url, - rank AS fts_rank${needsRatingJoin ? ",\n avg_r.avg_rating" : ""} - FROM chunks_fts f - JOIN documents d ON d.id = f.document_id${ - needsRatingJoin - ? ` - LEFT JOIN ( - SELECT document_id, AVG(rating) AS avg_rating - FROM ratings - GROUP BY document_id - ) avg_r ON avg_r.document_id = d.id` - : "" - } - WHERE chunks_fts MATCH ? - `; - orSql = appendFilters(orSql, orParams, options, "d"); + const rows = validateRows(Fts5RowSchema, db.prepare(sql).all(...params), label); + return { rows, baseSql, baseParams }; +} - // Update count base to use OR query - baseSql = orSql; - baseParams = [...orParams]; +/** FTS5-based full-text search with BM25 ranking. Uses AND logic by default. */ +function fts5Search( + db: Database.Database, + options: SearchOptions, + limit: number, + offset: number, +): SearchResponse { + // Sanitize and escape FTS5 query: strip dangerous syntax, wrap each word in quotes. + // AND-by-default: require all terms to match for better precision. + const words = options.query + .split(/\s+/) + .map((w) => sanitizeFtsWord(w)) + .filter((w) => w.length > 0); + if (words.length === 0) return { results: [], totalCount: 0 }; - orSql += " ORDER BY rank LIMIT ? OFFSET ?"; - orParams.push(limit, offset); + const quoted = words.map((w) => quoteFtsWord(w)); + let fts = runFtsQuery(db, options, quoted.join(" AND "), limit, offset, "fts5Search.rows"); - rows = validateRows(Fts5RowSchema, db.prepare(orSql).all(...orParams), "fts5Search.orRows"); + // If AND returned nothing, retry with OR for recall (the lazy count then uses the OR query). + // Leave out function words: "how to authenticate" must not match every chunk containing "to". + if (fts.rows.length === 0 && words.length > 1) { + const meaningful = words.filter((w) => !STOPWORDS.has(w.toLowerCase())); + const orWords = meaningful.length > 0 ? meaningful : words; + const orQuery = orWords.map((w) => quoteFtsWord(w)).join(" OR "); + fts = runFtsQuery(db, options, orQuery, limit, offset, "fts5Search.orRows"); } const totalCount = lazyCount( db, - baseSql, - baseParams, + fts.baseSql, + fts.baseParams, offset, - rows.length, + fts.rows.length, limit, "FTS5 search count", ); return { totalCount, - results: rows.map((row) => { + results: fts.rows.map((row) => { const bm25Score = -row.fts_rank; - return { - documentId: row.document_id, - chunkId: row.chunk_id, - title: row.title, - content: row.chunk_content, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - url: row.url, - score: bm25Score, - avgRating: row.avg_rating ?? null, - scoreExplanation: { - method: "fts5" as SearchMethod, - rawScore: row.fts_rank, - boostFactors: [], - details: `FTS5 BM25 ranking: raw_rank=${row.fts_rank.toFixed(4)}, score=${bm25Score.toFixed(4)}`, - }, - }; + return toResult(row, bm25Score, { + method: "fts5" as SearchMethod, + rawScore: row.fts_rank, + boostFactors: [], + details: `FTS5 BM25 ranking: raw_rank=${row.fts_rank.toFixed(4)}, score=${bm25Score.toFixed(4)}`, + }); }), }; } diff --git a/src/core/spider.ts b/src/core/spider.ts index 56c5311f..91838ad5 100644 --- a/src/core/spider.ts +++ b/src/core/spider.ts @@ -9,9 +9,12 @@ * - Private/internal IPs are blocked by the underlying url-fetcher */ +import { setTimeout as sleep } from "node:timers/promises"; import { getLogger } from "../logger.js"; import { FetchError } from "../errors.js"; +import { firstMarkdownHeading } from "../utils/markdown.js"; import { fetchRaw, type FetchOptions } from "./url-fetcher.js"; +import { decodeHtmlEntities } from "./html-entities.js"; import { extractLinks } from "./link-extractor.js"; import { NodeHtmlMarkdown } from "node-html-markdown"; @@ -44,6 +47,8 @@ export interface SpiderOptions { FetchOptions, "allowPrivateUrls" | "allowSelfSignedCerts" | "timeout" | "maxBodySize" >; + /** Abort the crawl between pages; the generator then throws the signal's reason. */ + signal?: AbortSignal; } export interface SpiderResult { @@ -266,11 +271,11 @@ function stripTags(input: string): string { function extractTitle(html: string, url: string): string { // Try <title> tag const match = /<title[^>]*>([^<]+)<\/title>/i.exec(html); - if (match?.[1]) return match[1].trim(); + if (match?.[1]) return decodeHtmlEntities(match[1].trim()); // Try first <h1> const h1 = /<h1[^>]*>([\s\S]*?)<\/h1>/i.exec(html); if (h1?.[1]) { - return stripTags(h1[1]).trim(); + return decodeHtmlEntities(stripTags(h1[1]).trim()); } // Fall back to URL path try { @@ -451,6 +456,75 @@ async function fetchSpiderPage( } } +type QueueEntry = { url: string; depth: number }; + +/** Mutable state of one crawl (breadth-first queue, visited URLs, stats). */ +interface CrawlState { + config: SpiderConfig; + stats: SpiderStats; + robotsCache: Map<string, Set<string>>; + visited: Set<string>; + queue: QueueEntry[]; + deadline: number; + signal: AbortSignal | undefined; + log: ReturnType<typeof getLogger>; +} + +/** + * Take the next unvisited queue entry and mark it visited. Returns undefined when the crawl + * is over: the queue is empty, maxPages is reached, or the total timeout has passed. + */ +function takeNextEntry(crawl: CrawlState): QueueEntry | undefined { + const { queue, stats, config, visited } = crawl; + while (queue.length > 0 && stats.pagesFetched < config.maxPages) { + crawl.signal?.throwIfAborted(); + if (checkDeadline(crawl.deadline, stats, crawl.log)) return undefined; + const entry = queue.shift()!; + if (!visited.has(entry.url)) { + visited.add(entry.url); + return entry; + } + } + return undefined; +} + +/** Fetch queue entries until one gives a page; undefined when the crawl is over. */ +async function nextCrawledPage( + crawl: CrawlState, +): Promise<{ raw: Awaited<ReturnType<typeof fetchRaw>>; result: SpiderResult } | undefined> { + const entry = takeNextEntry(crawl); + if (!entry) return undefined; + const { url, depth } = entry; + const { config, stats, log } = crawl; + + if (depth > 0 && (await shouldSkipNonSeedUrl(url, config, crawl.robotsCache, stats, log))) { + return nextCrawledPage(crawl); + } + + const raw = await fetchSpiderPage(url, config, stats, log); + crawl.signal?.throwIfAborted(); + if (!raw) return nextCrawledPage(crawl); + + const result = convertPage(raw, url, depth); + if (result.url !== url) crawl.visited.add(result.url); + + stats.pagesFetched++; + return { raw, result }; +} + +/** Yield crawled pages breadth-first; each page's links are queued after it is consumed. */ +async function* crawlPages(crawl: CrawlState): AsyncGenerator<SpiderResult, void, unknown> { + const page = await nextCrawledPage(crawl); + if (!page) return; + const { raw, result } = page; + yield result; + + if (result.depth < crawl.config.maxDepth) { + enqueueChildLinks(raw, result.url, result.depth, crawl.visited, crawl.queue); + } + yield* crawlPages(crawl); +} + /** * Spider a seed URL, yielding each successfully fetched page as a SpiderResult. * Performs BFS up to maxDepth hops and maxPages total. @@ -479,33 +553,19 @@ export async function* spiderUrl( robotsCache.set(config.seedOrigin, seedRobots); log.debug({ origin: config.seedOrigin, rules: seedRobots.size }, "Loaded robots.txt rules"); - const visited = new Set<string>(); - type QueueEntry = { url: string; depth: number }; - const queue: QueueEntry[] = [{ url: seedUrl, depth: 0 }]; - const deadline = Date.now() + HARD_TOTAL_TIMEOUT_MS; - - while (queue.length > 0 && stats.pagesFetched < config.maxPages) { - if (checkDeadline(deadline, stats, log)) break; - - const { url, depth } = queue.shift()!; - if (visited.has(url)) continue; - visited.add(url); - - if (depth > 0 && (await shouldSkipNonSeedUrl(url, config, robotsCache, stats, log))) continue; - - const raw = await fetchSpiderPage(url, config, stats, log); - if (!raw) continue; - - const result = convertPage(raw, url, depth); - if (result.url !== url) visited.add(result.url); + const crawl: CrawlState = { + config, + stats, + robotsCache, + visited: new Set<string>(), + queue: [{ url: seedUrl, depth: 0 }], + deadline: Date.now() + HARD_TOTAL_TIMEOUT_MS, + signal: options.signal, + log, + }; + const { queue } = crawl; - stats.pagesFetched++; - yield result; - - if (depth < config.maxDepth) { - enqueueChildLinks(raw, result.url, depth, visited, queue); - } - } + yield* crawlPages(crawl); if (!stats.abortReason && queue.length > 0 && stats.pagesFetched >= config.maxPages) { stats.abortReason = "maxPages"; @@ -527,8 +587,8 @@ export async function* spiderUrl( function extractTextTitle(text: string, url: string): string { // For plain text/markdown, try first # heading - const match = /^#\s+(.+)$/m.exec(text); - if (match?.[1]) return match[1].trim(); + const heading = firstMarkdownHeading(text); + if (heading) return heading.trim(); // Fall back to URL try { const parsed = new URL(url); @@ -540,7 +600,3 @@ function extractTextTitle(text: string, url: string): string { return url; } } - -function sleep(ms: number): Promise<void> { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/src/core/stopwords.ts b/src/core/stopwords.ts new file mode 100644 index 00000000..2ff88ea4 --- /dev/null +++ b/src/core/stopwords.ts @@ -0,0 +1,120 @@ +/** + * English function words that carry no meaning on their own: left out of tag suggestions, and + * out of the any-word keyword search that runs when no chunk contains every query word. + */ +export const STOPWORDS: ReadonlySet<string> = new Set([ + "the", + "a", + "an", + "is", + "are", + "was", + "were", + "be", + "been", + "being", + "have", + "has", + "had", + "do", + "does", + "did", + "will", + "would", + "shall", + "should", + "may", + "might", + "must", + "can", + "could", + "that", + "this", + "with", + "from", + "for", + "not", + "but", + "and", + "or", + "nor", + "so", + "yet", + "both", + "either", + "neither", + "each", + "every", + "all", + "any", + "few", + "more", + "most", + "other", + "some", + "such", + "than", + "too", + "very", + "just", + "about", + "above", + "after", + "again", + "against", + "below", + "between", + "during", + "into", + "through", + "under", + "until", + "also", + "how", + "what", + "when", + "where", + "which", + "while", + "who", + "whom", + "why", + "its", + "our", + "their", + "your", + "his", + "her", + "out", + "then", + "there", + "these", + "those", + "them", + "they", + "you", + "only", + "own", + "same", + "here", + "over", + "once", + "use", + "used", + "to", + "of", + "in", + "on", + "at", + "by", + "as", + "it", + "if", + "we", + "i", + "me", + "my", + "am", + "no", + "up", +]); diff --git a/src/core/tags.ts b/src/core/tags.ts index 2f85b699..007a669e 100644 --- a/src/core/tags.ts +++ b/src/core/tags.ts @@ -4,7 +4,8 @@ import { z } from "zod"; import { DocumentNotFoundError, ValidationError } from "../errors.js"; import { validateRow, validateRows } from "../db/validate.js"; import { createChildLogger } from "../logger.js"; -import type { Document } from "./documents.js"; +import { rowToDocument, type Document } from "./documents.js"; +import { STOPWORDS } from "./stopwords.js"; const TagRowSchema = z.object({ id: z.string(), @@ -48,109 +49,6 @@ const TitleContentRowSchema = z.object({ content: z.string(), }); -const STOPWORDS = new Set([ - "the", - "a", - "an", - "is", - "are", - "was", - "were", - "be", - "been", - "being", - "have", - "has", - "had", - "do", - "does", - "did", - "will", - "would", - "shall", - "should", - "may", - "might", - "must", - "can", - "could", - "that", - "this", - "with", - "from", - "for", - "not", - "but", - "and", - "or", - "nor", - "so", - "yet", - "both", - "either", - "neither", - "each", - "every", - "all", - "any", - "few", - "more", - "most", - "other", - "some", - "such", - "than", - "too", - "very", - "just", - "about", - "above", - "after", - "again", - "against", - "below", - "between", - "during", - "into", - "through", - "under", - "until", - "also", - "how", - "what", - "when", - "where", - "which", - "while", - "who", - "whom", - "why", - "its", - "our", - "their", - "your", - "his", - "her", - "our", - "out", - "then", - "there", - "these", - "those", - "them", - "they", - "you", - "your", - "only", - "own", - "same", - "here", - "over", - "once", - "use", - "used", -]); - export interface Tag { id: string; name: string; @@ -249,18 +147,28 @@ export function addTagsToDocument( return tags; } -/** Remove a specific tag from a document. */ -export function removeTagFromDocument( +/** + * Remove tags (by name, case-insensitive) from a document. Names the document does not carry + * are ignored. Returns the names that were removed. + */ +export function removeTagsFromDocument( db: Database.Database, documentId: string, - tagId: string, -): void { - const log = createChildLogger({ operation: "removeTagFromDocument" }); - db.prepare("DELETE FROM document_tags WHERE document_id = ? AND tag_id = ?").run( - documentId, - tagId, + tagNames: string[], +): string[] { + const log = createChildLogger({ operation: "removeTagsFromDocument" }); + const remove = db.prepare( + "DELETE FROM document_tags WHERE document_id = ? AND tag_id = (SELECT id FROM tags WHERE name = ?)", ); - log.info({ documentId, tagId }, "Tag removed from document"); + const removed: string[] = []; + const run = db.transaction(() => { + for (const name of new Set(tagNames.map((t) => t.trim().toLowerCase()))) { + if (remove.run(documentId, name).changes > 0) removed.push(name); + } + }); + run(); + log.info({ documentId, removed }, "Tags removed from document"); + return removed; } /** Get all tags for multiple documents in a single query. Returns a Map of documentId → tags. */ @@ -354,20 +262,7 @@ export function getDocumentsByTag( log.info({ tagNames: normalized, resultCount: rows.length }, "Documents retrieved by tags"); - return rows.map((row) => ({ - id: row.id, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - title: row.title, - content: row.content, - url: row.url, - contentHash: row.content_hash, - submittedBy: row.submitted_by, - createdAt: row.created_at, - updatedAt: row.updated_at, - })); + return rows.map((row) => rowToDocument(row)); } /** Tokenize text into lowercase words, filtering stopwords and short words. */ diff --git a/src/core/tasks.ts b/src/core/tasks.ts new file mode 100644 index 00000000..8ded7e0b --- /dev/null +++ b/src/core/tasks.ts @@ -0,0 +1,161 @@ +/** + * In-process registry of background tasks (long-running work started by MCP tools, REST + * routes or operations). Tasks are kept for one hour after they finish. + */ +import { randomUUID } from "node:crypto"; + +export type TaskStatus = "pending" | "running" | "completed" | "failed" | "cancelled"; +export type TaskType = + | "index_document" + | "reindex_documents" + | "sync_connector" + | "install_pack" + /** Any operation from src/core/operations run in the background; see Task.operation. */ + | "operation"; + +export interface TaskProgress { + current: number; + total: number; + message?: string | undefined; +} + +export interface Task { + id: string; + type: TaskType; + /** Operation name when `type` is "operation". */ + operation?: string | undefined; + status: TaskStatus; + progress?: TaskProgress | undefined; + result?: string | undefined; + error?: string | undefined; + createdAt: Date; + startedAt?: Date | undefined; + completedAt?: Date | undefined; +} + +/** Background work for a task: receives the task's abort signal and a progress reporter. */ +export type TaskWork = ( + signal: AbortSignal, + onProgress: (current: number, total: number) => void, +) => Promise<string>; + +/** True when `err` is the rejection produced by aborting `signal`. */ +export function isAbortRejection(err: unknown, signal: AbortSignal): boolean { + if (!signal.aborted) return false; + return err === signal.reason || (err instanceof Error && err.name === "AbortError"); +} + +/** TTL for completed/failed/cancelled tasks before they are pruned (1 hour). */ +const TASK_TTL_MS = 60 * 60 * 1000; + +export class TaskRegistry { + private readonly tasks = new Map<string, Task>(); + private readonly controllers = new Map<string, AbortController>(); + + /** Create a new task and return it along with its AbortSignal. */ + create(type: TaskType, operation?: string): { task: Task; signal: AbortSignal } { + const id = randomUUID(); + const task: Task = { + id, + type, + ...(operation === undefined ? {} : { operation }), + status: "pending", + createdAt: new Date(), + }; + const controller = new AbortController(); + this.tasks.set(id, task); + this.controllers.set(id, controller); + return { task, signal: controller.signal }; + } + + /** + * Create a task and run `work` in the background. + * Final status: "completed" if `work` resolves (even when cancellation was requested too late + * to stop it), "cancelled" if it rejects because the task signal was aborted, else "failed". + * `done` settles after the final status is recorded; it never rejects. + */ + run(type: TaskType, work: TaskWork, operation?: string): { task: Task; done: Promise<void> } { + const { task, signal } = this.create(type, operation); + this.update(task.id, { status: "running", startedAt: new Date() }); + const onProgress = (current: number, total: number): void => { + this.update(task.id, { progress: { current, total } }); + }; + // An async wrapper turns a synchronous throw from `work` into a rejection. + const start = async (): Promise<string> => await work(signal, onProgress); + const done = start().then( + (result) => { + this.update(task.id, { status: "completed", completedAt: new Date(), result }); + }, + (err: unknown) => { + this.update( + task.id, + isAbortRejection(err, signal) + ? { status: "cancelled", completedAt: new Date() } + : { + status: "failed", + completedAt: new Date(), + error: err instanceof Error ? err.message : String(err), + }, + ); + }, + ); + return { task, done }; + } + + /** Retrieve a task by ID. Returns undefined if not found or expired. */ + get(id: string): Task | undefined { + this.prune(); + return this.tasks.get(id); + } + + /** All tasks that have not expired, newest first. */ + list(): Task[] { + this.prune(); + return [...this.tasks.values()].sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()); + } + + /** Apply partial updates to a task. No-op if task not found. */ + update(id: string, updates: Partial<Task>): void { + const task = this.tasks.get(id); + if (task) { + Object.assign(task, updates); + } + } + + /** + * Attempt to cancel a task. + * Returns: + * "cancelled" — cancellation was requested + * "not_found" — task ID unknown or expired + * "already_terminal" — task already completed, failed, or cancelled + */ + cancel(id: string): "cancelled" | "not_found" | "already_terminal" { + this.prune(); + const task = this.tasks.get(id); + if (!task) return "not_found"; + if (task.status === "completed" || task.status === "failed" || task.status === "cancelled") { + return "already_terminal"; + } + this.controllers.get(id)?.abort(); + if (task.status === "pending") { + task.status = "cancelled"; + task.completedAt = new Date(); + } + // Running tasks detect abort via signal and update their own status. + return "cancelled"; + } + + /** Remove expired completed/failed/cancelled tasks. */ + private prune(): void { + const cutoff = Date.now() - TASK_TTL_MS; + for (const [id, task] of this.tasks) { + if (task.completedAt && task.completedAt.getTime() < cutoff) { + this.tasks.delete(id); + this.controllers.delete(id); + } + } + } +} + +/** Module-level singleton task registry shared by the MCP server, REST API and operations. */ +export const taskRegistry = new TaskRegistry(); diff --git a/src/core/topics.ts b/src/core/topics.ts index 59eb2afc..e053c543 100644 --- a/src/core/topics.ts +++ b/src/core/topics.ts @@ -3,7 +3,13 @@ import { randomUUID } from "node:crypto"; import { ValidationError, TopicNotFoundError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { validateRow } from "../utils/db-validation.js"; -import type { Document } from "./documents.js"; +import { + DOC_COLUMNS, + deleteChunkEmbeddings, + rowToDocument, + type Document, + type DocumentRow, +} from "./documents.js"; export interface Topic { id: string; @@ -14,6 +20,28 @@ export interface Topic { updatedAt: string; } +const TOPIC_COLUMNS = "id, name, description, parent_id, created_at, updated_at"; + +type TopicRow = { + id: string; + name: string; + description: string | null; + parent_id: string | null; + created_at: string; + updated_at: string; +}; + +function rowToTopic(row: TopicRow): Topic { + return { + id: row.id, + name: row.name, + description: row.description, + parentId: row.parent_id, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + export interface CreateTopicInput { name: string; description?: string | undefined; @@ -69,41 +97,20 @@ export function createTopic(db: Database.Database, input: CreateTopicInput): Top // Topic already existed — fetch and return it log.info({ name: input.name }, "Topic already exists, returning existing"); - const row = db - .prepare( - "SELECT id, name, description, parent_id, created_at, updated_at FROM topics WHERE name = ?", - ) - .get(input.name); + const row = db.prepare(`SELECT ${TOPIC_COLUMNS} FROM topics WHERE name = ?`).get(input.name); - const validated = validateRow<{ - id: string; - name: string; - description: string | null; - parent_id: string | null; - created_at: string; - updated_at: string; - }>( + const validated = validateRow<TopicRow>( row, ["id", "name", "description", "parent_id", "created_at", "updated_at"], "existing topic lookup", ); - return { - id: validated.id, - name: validated.name, - description: validated.description, - parentId: validated.parent_id, - createdAt: validated.created_at, - updatedAt: validated.updated_at, - }; + return rowToTopic(validated); } /** List topics, optionally filtered by parent. */ export function listTopics(db: Database.Database, parentId?: string): Topic[] { - let sql = ` - SELECT id, name, description, parent_id, created_at, updated_at - FROM topics - `; + let sql = `SELECT ${TOPIC_COLUMNS} FROM topics`; const params: unknown[] = []; if (parentId === undefined) { @@ -115,55 +122,35 @@ export function listTopics(db: Database.Database, parentId?: string): Topic[] { sql += " ORDER BY name"; - const rows = db.prepare(sql).all(...params) as Array<{ - id: string; - name: string; - description: string | null; - parent_id: string | null; - created_at: string; - updated_at: string; - }>; + const rows = db.prepare(sql).all(...params) as TopicRow[]; - return rows.map((row) => ({ - id: row.id, - name: row.name, - description: row.description, - parentId: row.parent_id, - createdAt: row.created_at, - updatedAt: row.updated_at, - })); + return rows.map((row) => rowToTopic(row)); } /** Get a topic by ID. */ export function getTopic(db: Database.Database, topicId: string): Topic { - const row = db - .prepare( - ` - SELECT id, name, description, parent_id, created_at, updated_at - FROM topics WHERE id = ? - `, - ) - .get(topicId) as - | { - id: string; - name: string; - description: string | null; - parent_id: string | null; - created_at: string; - updated_at: string; - } + const row = db.prepare(`SELECT ${TOPIC_COLUMNS} FROM topics WHERE id = ?`).get(topicId) as + | TopicRow | undefined; if (!row) throw new TopicNotFoundError(topicId); - return { - id: row.id, - name: row.name, - description: row.description, - parentId: row.parent_id, - createdAt: row.created_at, - updatedAt: row.updated_at, - }; + return rowToTopic(row); +} + +/** ID of the topic whose ID or name (case-insensitive) is `ref`, or undefined. */ +export function findTopicId(db: Database.Database, ref: string): string | undefined { + const row = db + .prepare("SELECT id FROM topics WHERE id = ? OR name = ? COLLATE NOCASE ORDER BY id = ? DESC") + .get(ref, ref, ref) as { id: string } | undefined; + return row?.id; +} + +/** ID of the topic whose ID or name is `ref`. Throws TopicNotFoundError when there is none. */ +export function resolveTopicId(db: Database.Database, ref: string): string { + const id = findTopicId(db, ref); + if (id === undefined) throw new TopicNotFoundError(ref); + return id; } export interface GetDocumentsByTopicOptions { @@ -179,7 +166,10 @@ export interface TopicStats { documentCount: number; } -/** Delete a topic and optionally its document associations. */ +/** Documents per DELETE statement when removing a topic's vectors (bounds SQL variables). */ +const EMBEDDING_DELETE_BATCH = 500; + +/** Delete a topic and optionally its documents (with their chunks and vectors). */ export function deleteTopic( db: Database.Database, topicId: string, @@ -189,6 +179,20 @@ export function deleteTopic( getTopic(db, topicId); const run = db.transaction(() => { if (options?.deleteDocuments) { + // chunk_embeddings is a virtual table with no foreign key, so remove vectors explicitly. + const docIds = ( + db.prepare("SELECT id FROM documents WHERE topic_id = ?").all(topicId) as Array<{ + id: string; + }> + ).map((row) => row.id); + try { + for (let i = 0; i < docIds.length; i += EMBEDDING_DELETE_BATCH) { + deleteChunkEmbeddings(db, docIds.slice(i, i + EMBEDDING_DELETE_BATCH)); + } + } catch (err: unknown) { + const tableMissing = err instanceof Error && err.message.includes("no such table"); + log[tableMissing ? "debug" : "warn"]({ err, topicId }, "Skipped chunk_embeddings cleanup"); + } db.prepare("DELETE FROM documents WHERE topic_id = ?").run(topicId); } db.prepare("DELETE FROM topics WHERE id = ?").run(topicId); @@ -223,37 +227,10 @@ export function getDocumentsByTopic( const offset = options?.offset ?? 0; const rows = db .prepare( - `SELECT id, source_type, library, version, topic_id, title, content, url, content_hash, submitted_by, created_at, updated_at - FROM documents WHERE topic_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`, + `SELECT ${DOC_COLUMNS} FROM documents WHERE topic_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`, ) - .all(topicId, limit, offset) as Array<{ - id: string; - source_type: string; - library: string | null; - version: string | null; - topic_id: string | null; - title: string; - content: string; - url: string | null; - content_hash: string | null; - submitted_by: string; - created_at: string; - updated_at: string; - }>; - return rows.map((row) => ({ - id: row.id, - sourceType: row.source_type, - library: row.library, - version: row.version, - topicId: row.topic_id, - title: row.title, - content: row.content, - url: row.url, - contentHash: row.content_hash, - submittedBy: row.submitted_by, - createdAt: row.created_at, - updatedAt: row.updated_at, - })); + .all(topicId, limit, offset) as DocumentRow[]; + return rows.map((row) => rowToDocument(row)); } /** Get topics with document counts. */ diff --git a/src/core/ttl.ts b/src/core/ttl.ts index 3b24c5c4..b8ce0e2b 100644 --- a/src/core/ttl.ts +++ b/src/core/ttl.ts @@ -1,5 +1,6 @@ import type Database from "better-sqlite3"; import { getLogger } from "../logger.js"; +import { deleteChunkEmbeddings } from "./documents.js"; export interface PruneResult { pruned: number; @@ -28,12 +29,7 @@ export function pruneExpiredDocuments(db: Database.Database): PruneResult { const pruneTransaction = db.transaction(() => { // Remove embeddings first (foreign key dependency) try { - db.prepare( - `DELETE FROM chunk_embeddings - WHERE chunk_id IN ( - SELECT id FROM chunks WHERE document_id IN (${placeholders}) - )`, - ).run(...ids); + deleteChunkEmbeddings(db, ids); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); if (msg.includes("no such table")) { diff --git a/src/core/url-fetcher.ts b/src/core/url-fetcher.ts index 2bbe7dd3..d2d1971e 100644 --- a/src/core/url-fetcher.ts +++ b/src/core/url-fetcher.ts @@ -3,11 +3,13 @@ import { promisify } from "node:util"; import { Agent } from "undici"; import { NodeHtmlMarkdown } from "node-html-markdown"; import { FetchError } from "../errors.js"; +import { firstMarkdownHeading } from "../utils/markdown.js"; import { getLogger } from "../logger.js"; +import { decodeHtmlEntities } from "./html-entities.js"; /** Lazy singleton undici Agent that skips TLS certificate verification. */ let _insecureAgent: Agent | undefined; -function getInsecureAgent(): Agent { +export function getInsecureAgent(): Agent { _insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } }); return _insecureAgent; } @@ -113,7 +115,7 @@ async function validateUrl(url: string, allowPrivateUrls = false): Promise<strin for (const addr of addresses) { if (!allowPrivateUrls && isPrivateIP(addr)) { throw new FetchError( - `Blocked request to private/internal IP ${addr} (resolved from ${hostname}). Set LIBSCOPE_ALLOW_PRIVATE_URLS=true to allow.`, + `Blocked request to private/internal IP ${addr} (resolved from ${hostname}). Set LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=true to allow.`, ); } } @@ -136,17 +138,21 @@ async function readBodyWithLimit(response: Response, limit: number): Promise<str const chunks: Uint8Array[] = []; let received = 0; - try { - let result = await reader.read(); - while (!result.done) { - const chunk = result.value; - received += chunk.byteLength; - if (received > limit) { - throw new FetchError(`Response body too large: exceeded ${limit} bytes`); - } - chunks.push(chunk); - result = await reader.read(); + // Read chunk by chunk until the stream ends, stopping once the limit is exceeded. + const readChunks = async (): Promise<void> => { + const result = await reader.read(); + if (result.done) return; + const chunk = result.value; + received += chunk.byteLength; + if (received > limit) { + throw new FetchError(`Response body too large: exceeded ${limit} bytes`); } + chunks.push(chunk); + return readChunks(); + }; + + try { + await readChunks(); } finally { await reader.cancel(); } @@ -160,21 +166,6 @@ async function readBodyWithLimit(response: Response, limit: number): Promise<str return new TextDecoder().decode(combined); } -/** Follow redirects manually so we can enforce a configurable limit. DNS-pinned to prevent rebinding. */ -async function fetchWithRedirects( - url: string, - timeout: number, - maxRedirects: number, - allowPrivateUrls: boolean, - allowSelfSignedCerts: boolean, -): Promise<Response> { - // Pass a per-request undici Agent when self-signed certs are allowed. - // This is scoped to this specific request chain and does not affect other - // concurrent requests (unlike mutating process.env["NODE_TLS_REJECT_UNAUTHORIZED"]). - const dispatcher = allowSelfSignedCerts ? getInsecureAgent() : undefined; - return _fetchWithRedirects(url, timeout, maxRedirects, allowPrivateUrls, dispatcher); -} - /** Re-resolve DNS after a fetch to detect DNS rebinding attacks. */ async function checkDnsRebinding(hostname: string, allowPrivateUrls: boolean): Promise<void> { const recheck = await Promise.allSettled([dns.resolve4(hostname), dns.resolve6(hostname)]); @@ -201,7 +192,8 @@ async function checkDnsRebinding(hostname: string, allowPrivateUrls: boolean): P } } -async function _fetchWithRedirects( +/** Follow redirects manually so we can enforce a configurable limit. DNS-pinned to prevent rebinding. */ +async function fetchWithRedirects( url: string, timeout: number, maxRedirects: number, @@ -266,12 +258,16 @@ export async function fetchRaw(url: string, options?: FetchOptions): Promise<Fet try { await validateUrl(url, allowPrivateUrls); + // Pass a per-request undici Agent when self-signed certs are allowed. + // This is scoped to this specific request chain and does not affect other + // concurrent requests (unlike mutating process.env["NODE_TLS_REJECT_UNAUTHORIZED"]). + const dispatcher = allowSelfSignedCerts ? getInsecureAgent() : undefined; const response = await fetchWithRedirects( url, timeout, maxRedirects, allowPrivateUrls, - allowSelfSignedCerts, + dispatcher, ); if (!response.ok) { @@ -279,20 +275,26 @@ export async function fetchRaw(url: string, options?: FetchOptions): Promise<Fet } const contentType = response.headers.get("content-type") ?? ""; + // Stream the body while enforcing actual byte-size limit const body = await readBodyWithLimit(response, maxBodySize); // Derive final URL from redirect chain (fetchWithRedirects resolves relative locations) const finalUrl = response.url ?? url; return { body, contentType, finalUrl }; } catch (err) { - if (err instanceof FetchError) throw err; - throw new FetchError( - `Failed to fetch URL: ${url} — ${err instanceof Error ? err.message : String(err)}`, - err, - ); + throw toFetchError(url, err); } } +/** Pass FetchErrors through unchanged; wrap anything else in a FetchError naming the URL. */ +function toFetchError(url: string, err: unknown): FetchError { + if (err instanceof FetchError) return err; + return new FetchError( + `Failed to fetch URL: ${url} — ${err instanceof Error ? err.message : String(err)}`, + err, + ); +} + /** * Fetch a URL and convert its HTML content to clean markdown-like text. * Strips tags, preserves code blocks and headings. @@ -304,29 +306,8 @@ export async function fetchAndConvert( const log = getLogger(); log.info({ url }, "Fetching URL"); - const { timeout, maxRedirects, maxBodySize, allowPrivateUrls, allowSelfSignedCerts } = { - ...DEFAULT_FETCH_OPTIONS, - ...options, - }; - try { - await validateUrl(url, allowPrivateUrls); - - const response = await fetchWithRedirects( - url, - timeout, - maxRedirects, - allowPrivateUrls, - allowSelfSignedCerts, - ); - - if (!response.ok) { - throw new FetchError(`HTTP ${response.status}: ${response.statusText}`); - } - - const contentType = response.headers.get("content-type") ?? ""; - // Stream the body while enforcing actual byte-size limit - const body = await readBodyWithLimit(response, maxBodySize); + const { body, contentType } = await fetchRaw(url, options); // If it's already markdown or plain text, return as-is if (contentType.includes("text/markdown") || contentType.includes("text/plain")) { @@ -343,24 +324,20 @@ export async function fetchAndConvert( content: converted, }; } catch (err) { - if (err instanceof FetchError) throw err; - throw new FetchError( - `Failed to fetch URL: ${url} — ${err instanceof Error ? err.message : String(err)}`, - err, - ); + throw toFetchError(url, err); } } /** Extract title from HTML <title> tag. */ function extractTitleFromHtml(html: string): string | null { const match = /<title[^>]*>([^<]+)<\/title>/i.exec(html); - return match?.[1]?.trim() ?? null; + const title = match?.[1]?.trim(); + return title ? decodeHtmlEntities(title) : null; } /** Extract title from first markdown heading. */ function extractTitleFromMarkdown(md: string): string | null { - const match = /^#\s+(.+)$/m.exec(md); - return match?.[1]?.trim() ?? null; + return firstMarkdownHeading(md)?.trim() ?? null; } /** Derive a title from the URL path. */ diff --git a/src/core/versioning.ts b/src/core/versioning.ts index 364a2735..5cdade69 100644 --- a/src/core/versioning.ts +++ b/src/core/versioning.ts @@ -2,7 +2,7 @@ import type Database from "better-sqlite3"; import { randomUUID } from "node:crypto"; import { z } from "zod"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { DocumentNotFoundError } from "../errors.js"; +import { NotFoundError } from "../errors.js"; import { getDocument, updateDocument } from "./documents.js"; import { getLogger } from "../logger.js"; import { validateRow, validateRows } from "../db/validate.js"; @@ -124,7 +124,10 @@ export function getVersion( .get(documentId, version); if (!raw) { - throw new DocumentNotFoundError(`Version ${version} of document ${documentId}`); + throw new NotFoundError( + `Version ${version} of document ${documentId} not found`, + "VERSION_NOT_FOUND", + ); } const row = validateRow(VersionRowSchema, raw, "getVersion"); diff --git a/src/core/watcher.ts b/src/core/watcher.ts index 5ac7ad9b..46621b16 100644 --- a/src/core/watcher.ts +++ b/src/core/watcher.ts @@ -3,10 +3,32 @@ import { extname, resolve, basename } from "node:path"; import { createHash } from "node:crypto"; import type Database from "better-sqlite3"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { indexDocument } from "./indexing.js"; +import { indexDocument, resolveSourceType, type SourceType } from "./indexing.js"; +import { deleteChunkEmbeddings } from "./documents.js"; +import { getParserForFile, getSupportedExtensions } from "./parsers/index.js"; import { createChildLogger } from "../logger.js"; -export const DEFAULT_WATCH_EXTENSIONS = [".md", ".mdx", ".txt", ".rst"]; +/** Extensions without a registered parser that the watcher indexes as plain UTF-8 text. */ +const PLAIN_TEXT_WATCH_EXTENSIONS = new Set([".rst"]); + +/** Every extension the parser registry supports, plus the plain-text extras. */ +export const DEFAULT_WATCH_EXTENSIONS = [ + ...getSupportedExtensions(), + ...PLAIN_TEXT_WATCH_EXTENSIONS, +]; + +/** + * Read a watched file and convert it to indexable text with the parser registry. + * Returns null when no parser handles the file's extension. + */ +export async function readWatchedFile(filePath: string): Promise<string | null> { + const parser = getParserForFile(filePath); + if (parser) return parser.parse(readFileSync(filePath)); + if (PLAIN_TEXT_WATCH_EXTENSIONS.has(extname(filePath).toLowerCase())) { + return readFileSync(filePath, "utf-8"); + } + return null; +} export interface WatchOptions { directory: string; @@ -15,6 +37,15 @@ export interface WatchOptions { onIndex?: (path: string) => void; onRemove?: (path: string) => void; onError?: (err: Error) => void; + /** Metadata for documents the watcher indexes (default: none, source type "manual"). */ + document?: + | { + topicId?: string | undefined; + library?: string | undefined; + version?: string | undefined; + sourceType?: SourceType | undefined; + } + | undefined; } export class FileWatcher { @@ -95,7 +126,11 @@ export class FileWatcher { if (!stat.isFile()) return; - const content = readFileSync(fullPath, "utf-8"); + const content = await readWatchedFile(fullPath); + if (content === null) { + this.log.warn({ path: fullPath }, "No parser for file extension, skipping"); + return; + } const contentHash = createHash("sha256").update(content).digest("hex"); const existing = this.db @@ -108,10 +143,14 @@ export class FileWatcher { } const title = basename(fullPath).replace(/\.[^.]+$/, ""); + const doc = this.options.document ?? {}; const result = await indexDocument(this.db, this.provider, { title, content, - sourceType: "manual", + sourceType: resolveSourceType({ ...doc, topic: doc.topicId }), + library: doc.library, + version: doc.version, + topicId: doc.topicId, url: fullPath, }); @@ -133,11 +172,7 @@ export class FileWatcher { if (!existing) return; try { - this.db - .prepare( - "DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)", - ) - .run(existing.id); + deleteChunkEmbeddings(this.db, [existing.id]); } catch { // chunk_embeddings table may not exist } diff --git a/src/core/webhooks.ts b/src/core/webhooks.ts index c4ae6705..29fc1e7a 100644 --- a/src/core/webhooks.ts +++ b/src/core/webhooks.ts @@ -9,9 +9,10 @@ import { import { promises as dns, lookup as dnsLookup } from "node:dns"; import { promisify } from "node:util"; import type Database from "better-sqlite3"; -import { ValidationError } from "../errors.js"; +import { NotFoundError, ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; import { isPrivateIP } from "./url-fetcher.js"; +import { LIBSCOPE_EVENTS, onEvent, type LibScopeEvent } from "./events.js"; const lookupAsync = promisify(dnsLookup); @@ -43,15 +44,9 @@ function decryptSecret(stored: string): string { return decipher.update(Buffer.from(encHex!, "hex")).toString("utf8") + decipher.final("utf8"); } -export const WEBHOOK_EVENTS = [ - "document.created", - "document.updated", - "document.deleted", - "document.rated", - "search.executed", -] as const; +export const WEBHOOK_EVENTS = LIBSCOPE_EVENTS; -export type WebhookEvent = (typeof WEBHOOK_EVENTS)[number]; +export type WebhookEvent = LibScopeEvent; export interface Webhook { id: string; @@ -68,8 +63,10 @@ export interface Webhook { * Payload sent to webhook subscribers. * * The `data` field shape varies by event type: - * - "document.created" / "document.updated" / "document.deleted": + * - "document.created" / "document.updated": * { documentId: string; title: string; library?: string; version?: string } + * - "document.deleted": + * { documentId: string } * - "document.rated": * { documentId: string; rating: number; feedback?: string } * - "search.executed": @@ -248,7 +245,7 @@ export function listWebhooks(db: Database.Database, limit?: number, offset?: num export function getWebhook(db: Database.Database, id: string): Webhook { const row = db.prepare("SELECT * FROM webhooks WHERE id = ?").get(id) as WebhookRow | undefined; if (!row) { - throw new ValidationError(`Webhook not found: ${id}`); + throw new NotFoundError(`Webhook not found: ${id}`, "WEBHOOK_NOT_FOUND"); } return rowToWebhook(row); } @@ -256,7 +253,7 @@ export function getWebhook(db: Database.Database, id: string): Webhook { export function deleteWebhook(db: Database.Database, id: string): void { const result = db.prepare("DELETE FROM webhooks WHERE id = ?").run(id); if (result.changes === 0) { - throw new ValidationError(`Webhook not found: ${id}`); + throw new NotFoundError(`Webhook not found: ${id}`, "WEBHOOK_NOT_FOUND"); } } @@ -318,6 +315,60 @@ export function buildPayload(event: WebhookEvent, data: Record<string, unknown>) return JSON.stringify(payload); } +const DELIVERY_TIMEOUT_MS = 5000; + +/** Payload data sent by the CLI and REST "test webhook" commands. */ +export const TEST_PING_DATA: Readonly<Record<string, unknown>> = Object.freeze({ + test: true, + message: "Webhook test ping", +}); + +/** + * Deliver one event to one webhook: decrypt the stored secret, sign the payload, + * re-check the URL against private IPs, and POST it (redirects are refused). + * Throws on SSRF rejection or network failure; returns the HTTP response otherwise. + */ +export async function deliverWebhook( + webhook: Webhook, + event: WebhookEvent, + data: Record<string, unknown>, +): Promise<Response> { + const body = buildPayload(event, data); + const headers: Record<string, string> = { "Content-Type": "application/json" }; + if (webhook.secret) { + headers["X-LibScope-Signature"] = signPayload(body, decryptSecret(webhook.secret)); + } + await validateWebhookUrlSsrf(webhook.url); + return fetch(webhook.url, { + method: "POST", + headers, + body, + redirect: "error", + signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS), + }); +} + +/** Record a delivery outcome; skipped when the caller already closed the database. */ +function recordDelivery( + db: Database.Database, + log: ReturnType<typeof getLogger>, + webhookId: string, + ok: boolean, +): void { + if (!db.open) return; + try { + if (ok) { + db.prepare( + "UPDATE webhooks SET last_triggered_at = datetime('now'), failure_count = 0 WHERE id = ?", + ).run(webhookId); + } else { + recordFailure(db, log, webhookId); + } + } catch (dbErr: unknown) { + log.error({ err: dbErr, webhookId }, "DB error recording webhook delivery"); + } +} + /** * Fire webhooks for a given event. Sends HTTP POST to all active webhooks * subscribed to this event. Fire-and-forget — errors are caught internally. @@ -334,56 +385,29 @@ export function fireWebhooks( ) .all() as WebhookRow[]; - const body = buildPayload(event, data); - for (const row of rows) { const webhook = rowToWebhook(row); if (!webhook.events.includes(event)) { continue; } - const headers: Record<string, string> = { - "Content-Type": "application/json", - }; - if (webhook.secret) { - headers["X-LibScope-Signature"] = signPayload(body, decryptSecret(webhook.secret)); - } - - // SSRF check before firing - validateWebhookUrlSsrf(webhook.url) - .then(() => - fetch(webhook.url, { - method: "POST", - headers, - body, - redirect: "error", - signal: AbortSignal.timeout(5000), - }), - ) - .then((resp) => { - try { - if (!resp.ok) { - log.warn( - { webhookId: webhook.id, url: webhook.url, status: resp.status }, - "Webhook delivery received non-2xx response", - ); - recordFailure(db, log, webhook.id); - return; - } - db.prepare( - "UPDATE webhooks SET last_triggered_at = datetime('now'), failure_count = 0 WHERE id = ?", - ).run(webhook.id); - } catch (dbErr: unknown) { - log.error({ err: dbErr, webhookId: webhook.id }, "DB error recording webhook success"); + deliverWebhook(webhook, event, data).then( + (resp) => { + if (!resp.ok) { + log.warn( + { webhookId: webhook.id, url: webhook.url, status: resp.status }, + "Webhook delivery received non-2xx response", + ); } - }) - .catch((err: unknown) => { + recordDelivery(db, log, webhook.id, resp.ok); + }, + (err: unknown) => { log.warn({ err, webhookId: webhook.id, url: webhook.url }, "Webhook delivery failed"); - try { - recordFailure(db, log, webhook.id); - } catch (dbErr: unknown) { - log.error({ err: dbErr, webhookId: webhook.id }, "DB error recording webhook failure"); - } - }); + recordDelivery(db, log, webhook.id, false); + }, + ); } } + +// Deliver core events (document/search mutations) to registered webhooks. +onEvent(fireWebhooks); diff --git a/src/core/workspace.ts b/src/core/workspace.ts index cfb2c142..0e113757 100644 --- a/src/core/workspace.ts +++ b/src/core/workspace.ts @@ -133,6 +133,11 @@ export function getWorkspacePath(name: string): string { return join(getWorkspacesDir(), name, "libscope.db"); } +/** Database location used by older releases (and by the SDK before workspaces applied to it). */ +export function getLegacyDatabasePath(): string { + return join(homedir(), ".libscope", "libscope.db"); +} + /** * Resolve the active workspace with precedence: * 1. LIBSCOPE_WORKSPACE env var diff --git a/src/db/connection.ts b/src/db/connection.ts index ba49aed9..355ee84c 100644 --- a/src/db/connection.ts +++ b/src/db/connection.ts @@ -4,17 +4,56 @@ import { createRequire } from "node:module"; import { dirname } from "node:path"; import { DatabaseError } from "../errors.js"; import { getLogger } from "../logger.js"; -import { getActiveWorkspace, getWorkspacePath } from "../core/workspace.js"; +import { expandHomeDir } from "../config.js"; +import { getActiveWorkspace, getLegacyDatabasePath, getWorkspacePath } from "../core/workspace.js"; const require = createRequire(import.meta.url); let db: Database.Database | null = null; let cachedPath: string | null = null; +let legacyWarningShown = false; + +export interface ResolveDatabasePathOptions { + /** Explicit database file (SDK `dbPath`, or `database.path` set in a config file). Wins over workspaces. */ + explicitPath?: string | undefined; + /** Workspace name. Defaults to the active workspace (LIBSCOPE_WORKSPACE, project file, `workspace use`). */ + workspace?: string | undefined; + /** Where to report the legacy-database notice (default: logger.warn). Called at most once per process. */ + warn?: ((message: string) => void) | undefined; +} + +/** + * The one rule for where the database lives, used by the CLI, MCP server, REST API and SDK: + * explicit path > workspace (option, else the active workspace). + * + * If no explicit path is set, the workspace database does not exist yet, and a database exists at + * the pre-workspace location (~/.libscope/libscope.db), a single warning names that file and how to + * keep using it. Nothing is moved or deleted. + */ +export function resolveDatabasePath(options: ResolveDatabasePathOptions = {}): string { + if (options.explicitPath) return expandHomeDir(options.explicitPath); + + const resolved = getWorkspacePath(options.workspace ?? getActiveWorkspace()); + const legacy = getLegacyDatabasePath(); + if (!legacyWarningShown && !existsSync(resolved) && existsSync(legacy)) { + legacyWarningShown = true; + const message = + `Using a new database at ${resolved}. An existing database was found at ${legacy} ` + + `(the location used by earlier versions). To keep using it, run: ` + + `libscope config set database.path ${legacy}`; + (options.warn ?? ((m: string): void => getLogger().warn(m)))(message); + } + return resolved; +} /** Resolve the database path, falling back to the active workspace. */ export function resolveDbPath(dbPath?: string): string { - if (dbPath) return dbPath; - return getWorkspacePath(getActiveWorkspace()); + return resolveDatabasePath({ explicitPath: dbPath }); +} + +/** Reset the once-per-process legacy-database warning (for tests). */ +export function resetLegacyDatabaseWarning(): void { + legacyWarningShown = false; } /** Get or create the database connection. */ diff --git a/src/db/index-meta.ts b/src/db/index-meta.ts new file mode 100644 index 00000000..e20a580b --- /dev/null +++ b/src/db/index-meta.ts @@ -0,0 +1,219 @@ +import type Database from "better-sqlite3"; +import { ConfigError, DatabaseError, EmbeddingError } from "../errors.js"; +import { getLogger } from "../logger.js"; +import { MAX_EMBEDDING_DIMENSIONS } from "../providers/dimensions.js"; +import type { EmbeddingProvider } from "../providers/embedding.js"; + +/** Key/value table that records facts about how the indexes were built. */ +export const INDEX_META_DDL = ` + CREATE TABLE IF NOT EXISTS index_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) +`; + +/** The guidance shown whenever the vector index does not match the embedding model. */ +export const REBUILD_VECTOR_INDEX_HINT = + "Run `libscope admin reindex --rebuild` to rebuild the vector index with the configured embedding model."; + +/** + * sqlite-vec error raised when a vector's size differs from the vector table's + * (the embedding model changed after the index was built). Both spellings occur in sqlite-vec. + */ +export function isVectorDimensionError(err: unknown): boolean { + const msg = (err instanceof Error ? err.message : String(err)).toLowerCase(); + return msg.includes("dimension mismatch") || msg.includes("dimension mistmatch"); +} + +/** The embedding provider, model and vector size that built (or will build) the vector index. */ +export interface EmbeddingIndexIdentity { + provider?: string | undefined; + model?: string | undefined; + dimensions?: number | undefined; +} + +type IdentityKey = keyof EmbeddingIndexIdentity; + +const META_KEYS: Record<IdentityKey, string> = { + provider: "embedding_provider", + model: "embedding_model", + dimensions: "embedding_dimensions", +}; + +const IDENTITY_KEYS: IdentityKey[] = ["provider", "model", "dimensions"]; + +/** A vector size, or undefined when it is 0, negative or not a number (unknown). */ +function knownSize(dimensions: number | undefined): number | undefined { + return dimensions !== undefined && dimensions > 0 ? dimensions : undefined; +} + +/** Build an identity from a provider, or from a bare vector size. */ +export function toEmbeddingIdentity( + source: number | Pick<EmbeddingProvider, "name" | "model" | "dimensions">, +): EmbeddingIndexIdentity { + if (typeof source === "number") return { dimensions: source }; + return { provider: source.name, model: source.model, dimensions: source.dimensions }; +} + +/** Read the stored identity. Returns an empty object when nothing is recorded. */ +export function readEmbeddingIdentity(db: Database.Database): EmbeddingIndexIdentity { + let rows: Array<{ key: string; value: string }>; + try { + rows = db.prepare("SELECT key, value FROM index_meta").all() as Array<{ + key: string; + value: string; + }>; + } catch { + return {}; // index_meta does not exist before migration 18 + } + const values = new Map(rows.map((r) => [r.key, r.value])); + return { + provider: values.get(META_KEYS.provider), + model: values.get(META_KEYS.model), + dimensions: knownSize(Number(values.get(META_KEYS.dimensions))), + }; +} + +/** + * Store an identity. Fields that are undefined are removed when `replace` is true and + * left as they are otherwise. Only changed rows are written. + */ +export function writeEmbeddingIdentity( + db: Database.Database, + identity: EmbeddingIndexIdentity, + replace: boolean, +): void { + const stored = readEmbeddingIdentity(db); + const changed = IDENTITY_KEYS.filter( + (k) => identity[k] !== stored[k] && (replace || identity[k] !== undefined), + ); + if (changed.length === 0) return; + db.exec(INDEX_META_DDL); + const upsert = db.prepare( + "INSERT INTO index_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ); + const remove = db.prepare("DELETE FROM index_meta WHERE key = ?"); + for (const k of changed) { + const value = identity[k]; + if (value === undefined) remove.run(META_KEYS[k]); + else upsert.run(META_KEYS[k], String(value)); + } +} + +/** + * Vector size of the existing chunk_embeddings table: undefined when the table does + * not exist, 0 when it exists but its size cannot be read from its definition. + */ +export function getVectorTableDimensions(db: Database.Database): number | undefined { + const row = db.prepare("SELECT sql FROM sqlite_master WHERE name = 'chunk_embeddings'").get() as + | { sql: string | null } + | undefined; + if (!row) return undefined; + const match = /float\[(\d+)\]/.exec(row.sql ?? ""); + return match ? Number(match[1]) : 0; +} + +/** Human-readable form, for example `ollama/nomic-embed-text, 768 dimensions`. */ +export function describeEmbeddingIdentity(identity: EmbeddingIndexIdentity): string { + const parts = [[identity.provider, identity.model].filter(Boolean).join("/")]; + if (identity.dimensions) parts.push(`${identity.dimensions} dimensions`); + return parts.filter(Boolean).join(", ") || "unknown"; +} + +/** True when a field is known on both sides and differs. */ +function identitiesConflict(a: EmbeddingIndexIdentity, b: EmbeddingIndexIdentity): boolean { + return IDENTITY_KEYS.some((k) => { + const left = a[k]; + const right = b[k]; + return Boolean(left) && Boolean(right) && left !== right; + }); +} + +/** + * Check the configured provider/model/vector size against the existing vector index and + * record what is known. Throws a ConfigError that names both when they differ. + */ +export function checkVectorIndexIdentity( + db: Database.Database, + configured: EmbeddingIndexIdentity, + tableDimensions: number, +): void { + const stored = readEmbeddingIdentity(db); + const index: EmbeddingIndexIdentity = { + ...stored, + dimensions: knownSize(tableDimensions) ?? stored.dimensions, + }; + if (identitiesConflict(index, configured)) { + throw new ConfigError( + `The vector index (${describeEmbeddingIdentity(index)}) does not match ` + + `the configured embedding model (${describeEmbeddingIdentity(configured)}). ` + + REBUILD_VECTOR_INDEX_HINT, + ); + } + writeEmbeddingIdentity( + db, + { + provider: configured.provider ?? index.provider, + model: configured.model ?? index.model, + dimensions: index.dimensions ?? knownSize(configured.dimensions), + }, + false, + ); +} + +/** Throw a DatabaseError unless `dimensions` is a usable vector size. */ +export function assertValidVectorDimensions(dimensions: number): void { + if (!Number.isInteger(dimensions) || dimensions <= 0 || dimensions > MAX_EMBEDDING_DIMENSIONS) { + throw new DatabaseError( + `Invalid vector dimensions: must be a positive integer <= ${MAX_EMBEDDING_DIMENSIONS}`, + ); + } +} + +/** Create chunk_embeddings with the given vector size (no-op if it already exists). */ +export function execCreateVectorTable(db: Database.Database, dimensions: number): void { + assertValidVectorDimensions(dimensions); + // dimensions is validated as a positive integer above, so interpolation is safe here + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS chunk_embeddings USING vec0( + chunk_id TEXT PRIMARY KEY, + embedding float[${dimensions}] + ); + `); +} + +/** + * Drop and recreate chunk_embeddings for the given provider, and record the provider, + * model and vector size. All stored vectors are removed; re-embed the chunks afterwards. + * One probe text is embedded first to learn the vector size and to confirm that the model + * works, so a misconfigured model fails before the existing vectors are dropped. + */ +export async function rebuildVectorTable( + db: Database.Database, + provider: EmbeddingProvider, +): Promise<EmbeddingIndexIdentity> { + const dimensions = (await provider.embed("dimension probe")).length; + if (provider.dimensions && provider.dimensions !== dimensions) { + throw new EmbeddingError( + `Expected embedding dimension ${provider.dimensions}, got ${dimensions} from ${describeEmbeddingIdentity(toEmbeddingIdentity(provider))}.`, + ); + } + assertValidVectorDimensions(dimensions); + const identity: EmbeddingIndexIdentity = { ...toEmbeddingIdentity(provider), dimensions }; + + const rebuild = db.transaction(() => { + db.exec("DROP TABLE IF EXISTS chunk_embeddings"); + execCreateVectorTable(db, dimensions); + writeEmbeddingIdentity(db, identity, true); + }); + try { + rebuild(); + } catch (err) { + throw new DatabaseError( + "Could not rebuild the vector table. The sqlite-vec extension must be available.", + err, + ); + } + getLogger().info({ ...identity }, "Vector table rebuilt"); + return identity; +} diff --git a/src/db/index.ts b/src/db/index.ts index 6cc81d94..b0e453d3 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -4,5 +4,6 @@ export { resetDatabase, createDatabase, resolveDbPath, + resolveDatabasePath, } from "./connection.js"; export { runMigrations, createVectorTable } from "./schema.js"; diff --git a/src/db/schema.ts b/src/db/schema.ts index 5b0fb613..5dd714b2 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,8 +1,24 @@ import type Database from "better-sqlite3"; import { DatabaseError } from "../errors.js"; import { getLogger } from "../logger.js"; - -const SCHEMA_VERSION = 17; +import type { EmbeddingProvider } from "../providers/embedding.js"; +import { + INDEX_META_DDL, + assertValidVectorDimensions, + checkVectorIndexIdentity, + execCreateVectorTable, + getVectorTableDimensions, + toEmbeddingIdentity, + writeEmbeddingIdentity, +} from "./index-meta.js"; + +const SCHEMA_VERSION = 19; + +/** Copy every chunk into chunks_fts, keyed by the chunk's rowid. */ +const FTS_BACKFILL_SQL = ` + INSERT INTO chunks_fts(rowid, content, chunk_id, document_id) + SELECT rowid, content, id, document_id FROM chunks; +`; const MIGRATIONS: Record<number, string> = { 1: ` @@ -172,6 +188,9 @@ const MIGRATIONS: Record<number, string> = { INSERT INTO schema_version (version) VALUES (8); `, + // connector_configs is unused: no CLI, MCP, REST or scheduler code reads or writes it. + // Connector settings live in ~/.libscope/connectors/<name>.json. Only the exported + // save/load/deleteDbConnectorConfig helpers in src/connectors/index.ts touch it. 9: ` CREATE TABLE IF NOT EXISTS connector_configs ( type TEXT PRIMARY KEY, @@ -269,15 +288,83 @@ const MIGRATIONS: Record<number, string> = { INSERT INTO schema_version (version) VALUES (17); `, + // Rebuild chunks_fts keyed by chunks.rowid. The old triggers matched rows on the + // UNINDEXED chunk_id column, so every chunk delete scanned the whole FTS table. + // Repopulating from chunks also drops duplicate rows left by earlier backfills. + 18: ` + DROP TRIGGER IF EXISTS chunks_ai; + DROP TRIGGER IF EXISTS chunks_ad; + DROP TRIGGER IF EXISTS chunks_au; + DROP TABLE IF EXISTS chunks_fts; + + CREATE VIRTUAL TABLE chunks_fts USING fts5( + content, + chunk_id UNINDEXED, + document_id UNINDEXED, + tokenize='porter unicode61' + ); + + -- The DELETE before each INSERT clears a stale row that may hold the same rowid. + CREATE TRIGGER chunks_ai AFTER INSERT ON chunks BEGIN + DELETE FROM chunks_fts WHERE rowid = new.rowid; + INSERT INTO chunks_fts(rowid, content, chunk_id, document_id) + VALUES (new.rowid, new.content, new.id, new.document_id); + END; + + CREATE TRIGGER chunks_ad AFTER DELETE ON chunks BEGIN + DELETE FROM chunks_fts WHERE rowid = old.rowid; + END; + + CREATE TRIGGER chunks_au AFTER UPDATE ON chunks BEGIN + DELETE FROM chunks_fts WHERE rowid = old.rowid; + INSERT INTO chunks_fts(rowid, content, chunk_id, document_id) + VALUES (new.rowid, new.content, new.id, new.document_id); + END; + + ${FTS_BACKFILL_SQL} + + -- Serves the title + length dedup lookup that runs on every index call. + CREATE INDEX IF NOT EXISTS idx_documents_title ON documents(title); + + -- Records the embedding provider, model and vector size of chunk_embeddings. + ${INDEX_META_DDL}; + + INSERT INTO schema_version (version) VALUES (18); + `, + // Saved search filters use the search operation's field names: "source" -> "sourceType". + // Rows with invalid JSON or without "source" are left as they are. + 19: ` + UPDATE saved_searches + SET filters = json_remove( + json_set(filters, '$.sourceType', json_extract(filters, '$.source')), + '$.source' + ) + WHERE CASE + WHEN json_valid(filters) THEN json_type(filters, '$.source') IS NOT NULL + AND json_type(filters, '$.sourceType') IS NULL + ELSE 0 + END; + + INSERT INTO schema_version (version) VALUES (19); + `, }; -const FTS_BACKFILL_SQL = ` - INSERT INTO chunks_fts(content, chunk_id, document_id) - SELECT content, id, document_id FROM chunks; -`; +/** + * Populate chunks_fts from chunks, but only when the FTS index is empty (for example, + * right after it was created). A non-empty index is kept in sync by triggers, so + * backfilling it again would duplicate every row. + */ +function backfillFtsIfEmpty(db: Database.Database): void { + const hasFtsRows = db.prepare("SELECT 1 FROM chunks_fts LIMIT 1").get() !== undefined; + if (hasFtsRows) return; + db.exec(FTS_BACKFILL_SQL); +} -/** Run pending migrations on the database. */ -export function runMigrations(db: Database.Database): void { +/** + * Run pending migrations on the database. + * @param targetVersion - Stop at this schema version (defaults to the latest). Used by tests. + */ +export function runMigrations(db: Database.Database, targetVersion: number = SCHEMA_VERSION): void { const log = getLogger(); try { @@ -294,15 +381,15 @@ export function runMigrations(db: Database.Database): void { currentVersion = row?.version ?? 0; } - if (currentVersion >= SCHEMA_VERSION) { + if (currentVersion >= targetVersion) { log.debug({ currentVersion }, "Database schema is up to date"); return; } - log.info({ from: currentVersion, to: SCHEMA_VERSION }, "Running database migrations"); + log.info({ from: currentVersion, to: targetVersion }, "Running database migrations"); const migrate = db.transaction(() => { - for (let v = currentVersion + 1; v <= SCHEMA_VERSION; v++) { + for (let v = currentVersion + 1; v <= targetVersion; v++) { const sql = MIGRATIONS[v]; if (!sql) { throw new DatabaseError(`Missing migration for version ${v}`); @@ -315,7 +402,7 @@ export function runMigrations(db: Database.Database): void { migrate(); try { - db.exec(FTS_BACKFILL_SQL); + backfillFtsIfEmpty(db); } catch (err) { log.warn({ err }, "FTS backfill failed — new chunks will still be indexed via triggers"); } @@ -327,20 +414,40 @@ export function runMigrations(db: Database.Database): void { } } -/** Create the virtual table for vector search (requires sqlite-vec). */ -export function createVectorTable(db: Database.Database, dimensions: number): void { - if (!Number.isInteger(dimensions) || dimensions <= 0 || dimensions > 10000) { - throw new DatabaseError("Invalid vector dimensions: must be a positive integer <= 10000"); +/** + * Create the vector table (requires sqlite-vec) and record which embedding model it is for. + * Pass the provider rather than a bare vector size so that a change of provider or model + * is detected, not only a change of vector size. + * + * @throws ConfigError when the existing vector table was built for a different provider, + * model or vector size. The message tells the user to run `libscope admin reindex --rebuild`. + * @throws DatabaseError when the vector size is invalid, or unknown (0) and no table exists. + */ +export function createVectorTable( + db: Database.Database, + source: number | Pick<EmbeddingProvider, "name" | "model" | "dimensions">, +): void { + const configured = toEmbeddingIdentity(source); + const dimensions = configured.dimensions ?? 0; + // 0 means the provider learns its vector size from its first embedding. + if (dimensions !== 0) assertValidVectorDimensions(dimensions); + + const tableDimensions = getVectorTableDimensions(db); + if (tableDimensions !== undefined) { + checkVectorIndexIdentity(db, configured, tableDimensions); + return; } + if (dimensions === 0) { + throw new DatabaseError( + "Invalid vector dimensions: the vector size of the embedding model is not known yet. " + + "Set embedding.dimensions in the config, or run `libscope admin reindex --rebuild` to detect it.", + ); + } + const log = getLogger(); try { - // dimensions is validated as a positive integer above, so interpolation is safe here - db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS chunk_embeddings USING vec0( - chunk_id TEXT PRIMARY KEY, - embedding float[${dimensions}] - ); - `); + execCreateVectorTable(db, dimensions); + writeEmbeddingIdentity(db, configured, true); log.info({ dimensions }, "Vector table ready"); } catch (err) { log.warn({ err }, "Could not create vector table — vector search unavailable"); diff --git a/src/errors.ts b/src/errors.ts index bd141127..4d91985b 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -49,21 +49,32 @@ export class ConfigError extends LibScopeError { } } -export class DocumentNotFoundError extends LibScopeError { +/** + * A requested resource does not exist. REST maps every NotFoundError to HTTP 404. + * Subclasses keep their own `code`; other resources use this class directly with a specific code. + */ +export class NotFoundError extends LibScopeError { + constructor(message: string, code: string = "NOT_FOUND", cause?: unknown) { + super(message, code, cause); + this.name = "NotFoundError"; + } +} + +export class DocumentNotFoundError extends NotFoundError { constructor(documentId: string) { super(`Document not found: ${documentId}`, "DOCUMENT_NOT_FOUND"); this.name = "DocumentNotFoundError"; } } -export class ChunkNotFoundError extends LibScopeError { +export class ChunkNotFoundError extends NotFoundError { constructor(chunkId: string) { super(`Chunk not found: ${chunkId}`, "CHUNK_NOT_FOUND"); this.name = "ChunkNotFoundError"; } } -export class TopicNotFoundError extends LibScopeError { +export class TopicNotFoundError extends NotFoundError { constructor(topicId: string) { super(`Topic not found: ${topicId}`, "TOPIC_NOT_FOUND"); this.name = "TopicNotFoundError"; diff --git a/src/lite/chunker-treesitter.ts b/src/lite/chunker-treesitter.ts index 4c5af272..eeb56bdf 100644 --- a/src/lite/chunker-treesitter.ts +++ b/src/lite/chunker-treesitter.ts @@ -322,7 +322,7 @@ export class TreeSitterChunker { } catch (err: unknown) { throw new ValidationError( 'Code chunking requires the "tree-sitter" package. ' + - "Install it with: npm install tree-sitter tree-sitter-typescript tree-sitter-javascript tree-sitter-python", + "Install it with: npm install tree-sitter@0.21 tree-sitter-typescript@0.21 tree-sitter-javascript@0.21 tree-sitter-python@0.21", err, ); } @@ -341,24 +341,17 @@ export class TreeSitterChunker { try { const mod = (await import(packageName)) as Record<string, unknown>; - // Grammar packages typically export the language as the default export. - // tree-sitter-typescript exports { typescript, tsx } as named exports. - let grammar: unknown; - if (language === "typescript" && "typescript" in mod) { - grammar = mod["typescript"]; - } else if ("default" in mod) { - grammar = mod["default"]; - } else { - // Fallback: use the module itself (some packages export the grammar directly) - grammar = mod; - } + // The grammar packages are CommonJS: import() returns their module.exports as `default`. + const exported = ("default" in mod ? mod["default"] : mod) as Record<string, unknown>; + // tree-sitter-typescript exports { typescript, tsx }; the others export the language. + const grammar = language === "typescript" ? exported["typescript"] : exported; this.grammarCache.set(language, grammar); return grammar; } catch (err: unknown) { throw new ValidationError( `Code chunking for ${language} requires the "${packageName}" package. ` + - `Install it with: npm install ${packageName}`, + `Install it with: npm install ${packageName}@${language === "cpp" ? "0.22" : "0.21"}`, err, ); } diff --git a/src/lite/core.ts b/src/lite/core.ts index f3711125..6be6ada9 100644 --- a/src/lite/core.ts +++ b/src/lite/core.ts @@ -1,211 +1,62 @@ -import Database from "better-sqlite3"; -import { homedir } from "node:os"; -import { join } from "node:path"; +import { extname } from "node:path"; +import type Database from "better-sqlite3"; +import { ValidationError } from "../errors.js"; +import { LibScope } from "../LibScope.js"; +import type { ConfigOverrides } from "../core/bootstrap.js"; +import type { Chunker } from "../core/indexing.js"; +import type { LlmProvider } from "../core/rag.js"; import type { EmbeddingProvider } from "../providers/embedding.js"; -import { LocalEmbeddingProvider } from "../providers/local.js"; -import { createDatabase } from "../db/connection.js"; -import { runMigrations, createVectorTable } from "../db/schema.js"; -import { indexDocument } from "../core/indexing.js"; -import { searchDocuments } from "../core/search.js"; -import { bulkDelete } from "../core/bulk.js"; -import { rateDocument } from "../core/ratings.js"; -import { askQuestion, getContextForQuestion, type LlmProvider } from "../core/rag.js"; -import { normalizeRawInput } from "./normalize.js"; import { TreeSitterChunker } from "./chunker-treesitter.js"; -import type { - LiteOptions, - LiteDoc, - RawInput, - LiteSearchOptions, - LiteSearchResult, - LiteContextOptions, - LiteAskOptions, -} from "./types.js"; -export class LibScopeLite { - private readonly db: Database.Database; - private readonly provider: EmbeddingProvider; - private readonly llmProvider: LlmProvider | null; - private _chunker: TreeSitterChunker | undefined; - private get chunker(): TreeSitterChunker { - this._chunker ??= new TreeSitterChunker(); - return this._chunker; - } - - constructor(opts: LiteOptions = {}) { - this.provider = opts.provider ?? new LocalEmbeddingProvider(); - this.llmProvider = opts.llmProvider ?? null; - - if (opts.db === undefined) { - const dbPath = opts.dbPath ?? join(homedir(), ".libscope", "lite.db"); - // createDatabase handles directory creation, WAL mode, pragmas, and sqlite-vec loading. - this.db = createDatabase(dbPath); - runMigrations(this.db); - // Create vector table best-effort (requires sqlite-vec to be loaded). - try { - createVectorTable(this.db, this.provider.dimensions); - } catch { - /* sqlite-vec not loaded — FTS5 search still works */ - } - } else { - // Caller-provided DB: skip all setup (migrations, extension loading, vector table). - this.db = opts.db; - } - } - - async index(docs: LiteDoc[]): Promise<void> { - for (const doc of docs) { - let preChunked: string[] | undefined; - - if (doc.language && this.chunker.supports(doc.language)) { - try { - const codeChunks = await this.chunker.chunk(doc.content, doc.language); - preChunked = codeChunks.map((c) => c.content); - } catch { - // tree-sitter not installed or parse failed — fall back to text chunker - } - } - - await indexDocument(this.db, this.provider, { - title: doc.title, - content: doc.content, - sourceType: doc.sourceType ?? "manual", - library: doc.library, - version: doc.version, - topicId: doc.topicId, - url: doc.url, - preChunked, - }); - } - } - - async indexRaw(input: RawInput): Promise<string[]> { - const normalized = await normalizeRawInput(input); - if (normalized.chunks !== undefined && normalized.chunks.length > 1) { - const ids: string[] = []; - for (let i = 0; i < normalized.chunks.length; i++) { - const chunk = normalized.chunks[i]!; - const result = await indexDocument(this.db, this.provider, { - title: `${normalized.title} (part ${String(i + 1)})`, - content: chunk, - sourceType: "manual", - }); - ids.push(result.id); - } - return ids; - } - const result = await indexDocument(this.db, this.provider, { - title: normalized.title, - content: normalized.content, - sourceType: "manual", - url: input.type === "url" ? input.url : undefined, - }); - return [result.id]; - } - - async indexBatch(docs: LiteDoc[], opts: { concurrency: number }): Promise<void> { - const concurrency = Math.max(1, opts.concurrency); - let activeCount = 0; - let idx = 0; - - await new Promise<void>((resolve, reject) => { - if (docs.length === 0) { - resolve(); - return; - } - - const runNext = (): void => { - while (activeCount < concurrency && idx < docs.length) { - const doc = docs[idx]; - if (!doc) break; - idx++; - activeCount++; - this.index([doc]) - .then(() => { - activeCount--; - if (idx >= docs.length && activeCount === 0) { - resolve(); - } else { - runNext(); - } - }) - .catch(reject); - } - }; - - runNext(); - }); - } - - async search(query: string, opts?: LiteSearchOptions): Promise<LiteSearchResult[]> { - const { results } = await searchDocuments(this.db, this.provider, { - query, - limit: opts?.limit ?? 10, - topic: opts?.topic, - library: opts?.library, - tags: opts?.tags, - diversity: opts?.diversity, - }); - return results.map((r) => ({ - docId: r.documentId, - chunkId: r.chunkId, - title: r.title, - content: r.content, - score: r.score, - url: r.url, - })); - } +export interface LiteOptions { + /** SQLite database file (":memory:" for an in-memory database). */ + dbPath?: string | undefined; + /** Use an already-open database instead of `dbPath`. `close()` leaves it open. */ + db?: Database.Database | undefined; + /** Embedding provider. Default: the local provider (Xenova/all-MiniLM-L6-v2). */ + provider?: EmbeddingProvider | undefined; + /** LLM for ask/askStream. Without one, ask throws a ConfigError. */ + llmProvider?: LlmProvider | undefined; + /** Custom chunker, e.g. createCodeChunker(). */ + chunker?: Chunker | undefined; + /** Config values over the defaults (config files and environment are not read). */ + config?: ConfigOverrides | undefined; +} - async getContext(question: string, opts?: LiteContextOptions): Promise<string> { - const { contextPrompt } = await getContextForQuestion(this.db, this.provider, { - question, - topK: opts?.topK ?? 5, - topic: opts?.topic, - library: opts?.library, - }); - return contextPrompt; +/** + * LibScope for embedding in an application: the same object and results as + * `LibScope.create`, but config files and LIBSCOPE_* variables are ignored and the database + * is the one you name. + */ +export function createLite(options: LiteOptions): LibScope { + if (options.dbPath === undefined && options.db === undefined) { + throw new ValidationError("createLite needs dbPath or db"); } + return LibScope.create({ ...options, useConfigFile: false }); +} - async ask(question: string, opts?: LiteAskOptions): Promise<string> { - const llm = opts?.llmProvider ?? this.llmProvider; - if (!llm) { - throw new Error("No LlmProvider configured. Pass llmProvider to constructor or ask() opts."); - } - const result = await askQuestion(this.db, this.provider, llm, { - question, - topK: opts?.topK ?? 5, - topic: opts?.topic, - library: opts?.library, - systemPrompt: opts?.systemPrompt, - }); - return result.answer; - } +function languageOf(name: string): string | undefined { + const ext = extname(name).slice(1).toLowerCase(); + return ext === "" ? undefined : ext; +} - async *askStream(question: string, opts?: LiteAskOptions): AsyncGenerator<string> { - const llm = opts?.llmProvider ?? this.llmProvider; - if (!llm) { - throw new Error("No LlmProvider configured."); +/** + * A Chunker that splits source code at function and class boundaries with tree-sitter (an + * optional peer dependency). The language is `language` if given, else the extension of the + * file path, URL or title. Other documents, and code when tree-sitter is not installed, use + * the built-in chunker. + */ +export function createCodeChunker(options: { language?: string | undefined } = {}): Chunker { + const chunker = new TreeSitterChunker(); + return async ({ content, title, source }) => { + const language = options.language ?? languageOf(source) ?? languageOf(title); + if (language === undefined || !chunker.supports(language)) return undefined; + try { + const chunks = await chunker.chunk(content, language); + return chunks.map((c) => c.content); + } catch { + // tree-sitter not installed or the parse failed: use the built-in chunker + return undefined; } - if (!llm.completeStream) { - throw new Error("This LlmProvider does not support streaming."); - } - const context = await this.getContext(question, opts); - yield* llm.completeStream(context, opts?.systemPrompt); - } - - rate(docId: string, score: number): void { - rateDocument(this.db, { documentId: docId, rating: score }); - } - - deleteByLibrary(library: string): void { - // bulkDelete caps at MAX_BATCH_SIZE=1000; loop until all cleared - let result; - do { - result = bulkDelete(this.db, { library }, false); - } while (result.affected > 0); - } - - close(): void { - this.db.close(); - } + }; } diff --git a/src/lite/index.ts b/src/lite/index.ts index 9a353460..5e78ea54 100644 --- a/src/lite/index.ts +++ b/src/lite/index.ts @@ -1,13 +1,10 @@ -export { LibScopeLite } from "./core.js"; -export { TreeSitterChunker } from "./chunker-treesitter.js"; -export type { CodeChunk } from "./chunker-treesitter.js"; -export type { - LiteOptions, - LiteDoc, - RawInput, - LiteSearchOptions, - LiteSearchResult, - LiteContextOptions, - LiteAskOptions, -} from "./types.js"; -export type { LlmProvider } from "../core/rag.js"; +/** + * `libscope/lite`: LibScope for embedding in an application. Everything from the package + * root, plus a preset that ignores config files, a tree-sitter code chunker and input + * normalization. Must not import the CLI, MCP server or connectors + * (tests/unit/lite-import-graph.test.ts). + */ +export * from "../core/index.js"; +export { createLite, createCodeChunker, type LiteOptions } from "./core.js"; +export { TreeSitterChunker, type CodeChunk } from "./chunker-treesitter.js"; +export { normalizeRawInput, type RawInput, type NormalizedInput } from "./normalize.js"; diff --git a/src/lite/normalize.ts b/src/lite/normalize.ts index 8366b027..480b5f8c 100644 --- a/src/lite/normalize.ts +++ b/src/lite/normalize.ts @@ -2,95 +2,46 @@ import { readFileSync } from "node:fs"; import { basename, extname } from "node:path"; import { getParserForFile } from "../core/parsers/index.js"; import { fetchAndConvert } from "../core/url-fetcher.js"; -import type { RawInput } from "./types.js"; + +/** Raw input that normalizeRawInput turns into a title and markdown content. */ +export type RawInput = + | { type: "file"; path: string; title?: string | undefined } + | { type: "url"; url: string; title?: string | undefined } + | { type: "text"; content: string; title: string } + | { type: "buffer"; buffer: Buffer; filename: string; title?: string | undefined }; export interface NormalizedInput { title: string; content: string; - chunks?: string[]; -} - -// Code extensions that trigger tree-sitter attempt -const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py"]); - -type TreeSitterChunkerType = import("./chunker-treesitter.js").TreeSitterChunker; -let treeSitterChunker: TreeSitterChunkerType | null = null; -let treeSitterLoaded = false; - -async function getTreeSitterChunker(): Promise<TreeSitterChunkerType | null> { - if (treeSitterLoaded) return treeSitterChunker; - treeSitterLoaded = true; - try { - const { TreeSitterChunker } = await import("./chunker-treesitter.js"); - treeSitterChunker = new TreeSitterChunker(); - } catch { - /* optional dep not installed — graceful fallback */ - } - return treeSitterChunker; } -function extToLang(ext: string): string { - const map: Record<string, string> = { - ts: "typescript", - tsx: "typescript", - js: "javascript", - jsx: "javascript", - mjs: "javascript", - cjs: "javascript", - py: "python", - }; - return map[ext.slice(1)] ?? ext.slice(1); +/** Parse a file's bytes with the registered parser for its extension, else as UTF-8 text. */ +async function normalizeBuffer( + buf: Buffer, + filename: string, + explicitTitle: string | undefined, +): Promise<NormalizedInput> { + const title = explicitTitle ?? basename(filename, extname(filename)); + const parser = getParserForFile(filename); + const content = parser ? await parser.parse(buf) : buf.toString("utf-8"); + return { title, content }; } +/** + * Turn a file, buffer, URL or text into `{ title, content }` for `scope.add({ title, content })`. + * Files and buffers use the parser for their extension (PDF, DOCX, HTML, ...); other files + * (e.g. source code) are read as UTF-8 text. + */ export async function normalizeRawInput(input: RawInput): Promise<NormalizedInput> { switch (input.type) { case "text": return { title: input.title, content: input.content }; - case "file": { - const ext = extname(input.path).toLowerCase(); - const buf = readFileSync(input.path); - const title = input.title ?? basename(input.path, ext); + case "file": + return normalizeBuffer(readFileSync(input.path), input.path, input.title); - if (CODE_EXTENSIONS.has(ext)) { - const chunker = await getTreeSitterChunker(); - const lang = extToLang(ext); - if (chunker?.supports(lang)) { - const codeChunks = await chunker.chunk(buf.toString("utf-8"), lang); - return { - title, - content: codeChunks[0]?.content ?? "", - chunks: codeChunks.map((c) => c.content), - }; - } - } - - const parser = getParserForFile(input.path); - const content = parser ? await parser.parse(buf) : buf.toString("utf-8"); - return { title, content }; - } - - case "buffer": { - const ext = extname(input.filename).toLowerCase(); - const title = input.title ?? basename(input.filename, ext); - - if (CODE_EXTENSIONS.has(ext)) { - const chunker = await getTreeSitterChunker(); - const lang = extToLang(ext); - if (chunker?.supports(lang)) { - const codeChunks = await chunker.chunk(input.buffer.toString("utf-8"), lang); - return { - title, - content: codeChunks[0]?.content ?? "", - chunks: codeChunks.map((c) => c.content), - }; - } - } - - const parser = getParserForFile(input.filename); - const content = parser ? await parser.parse(input.buffer) : input.buffer.toString("utf-8"); - return { title, content }; - } + case "buffer": + return normalizeBuffer(input.buffer, input.filename, input.title); case "url": { const fetched = await fetchAndConvert(input.url); diff --git a/src/lite/types.ts b/src/lite/types.ts deleted file mode 100644 index 600303b5..00000000 --- a/src/lite/types.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import type { LlmProvider } from "../core/rag.js"; - -export interface LiteOptions { - /** Path to SQLite database file. Defaults to ~/.libscope/lite.db. Use ':memory:' for in-memory. */ - dbPath?: string | undefined; - /** Pre-configured database instance. When provided, dbPath is ignored and no migrations or - * vector table setup are performed — caller is responsible for schema initialization. */ - db?: Database.Database | undefined; - provider?: EmbeddingProvider | undefined; - model?: string | undefined; - llmProvider?: LlmProvider | undefined; -} - -export interface LiteDoc { - title: string; - content: string; - url?: string | undefined; - sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; - library?: string | undefined; - version?: string | undefined; - topicId?: string | undefined; - language?: string | undefined; -} - -export type RawInput = - | { type: "file"; path: string; title?: string | undefined } - | { type: "url"; url: string; title?: string | undefined } - | { type: "text"; content: string; title: string } - | { type: "buffer"; buffer: Buffer; filename: string; title?: string | undefined }; - -export interface LiteSearchOptions { - limit?: number | undefined; - topic?: string | undefined; - library?: string | undefined; - tags?: string[] | undefined; - diversity?: number | undefined; -} - -export interface LiteSearchResult { - docId: string; - chunkId: string; - title: string; - content: string; - score: number; - url: string | null; -} - -export interface LiteContextOptions { - topK?: number | undefined; - topic?: string | undefined; - library?: string | undefined; -} - -export interface LiteAskOptions { - topK?: number | undefined; - topic?: string | undefined; - library?: string | undefined; - systemPrompt?: string | undefined; - llmProvider?: LlmProvider | undefined; -} diff --git a/src/logger.ts b/src/logger.ts index 5a16a9cc..c78daa23 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -3,11 +3,28 @@ import pino from "pino"; export type LogLevel = "debug" | "info" | "warn" | "error" | "silent"; -let currentLogger: pino.Logger = pino({ level: "info" }); +/** Where log lines are written. */ +export type LogDestination = "stdout" | "stderr"; -/** Initialize the logger with a specific level. */ -export function initLogger(level: LogLevel): void { - currentLogger = pino({ level }); +export interface LoggerOptions { + /** + * Output stream for log lines (default: "stderr"). + * Logs go to stderr by default so they never mix with program output on stdout + * (the MCP stdio transport, `--json` CLI output, piped results). + */ + destination?: LogDestination; +} + +function createLogger(level: LogLevel, destination: LogDestination): pino.Logger { + const fd = destination === "stdout" ? 1 : 2; + return pino({ level }, pino.destination({ dest: fd, sync: true })); +} + +let currentLogger: pino.Logger = createLogger("info", "stderr"); + +/** Initialize the logger with a specific level and output stream (stderr by default). */ +export function initLogger(level: LogLevel, options?: LoggerOptions): void { + currentLogger = createLogger(level, options?.destination ?? "stderr"); } /** Get the current logger instance. */ diff --git a/src/mcp/errors.ts b/src/mcp/errors.ts index 38f07b85..49278d23 100644 --- a/src/mcp/errors.ts +++ b/src/mcp/errors.ts @@ -26,6 +26,11 @@ export function errorResponse(err: unknown): { export type ToolResult = { content: Array<{ type: "text"; text: string }>; isError?: boolean }; +/** Wrap plain text as a single-item MCP tool result. */ +export function textResult(text: string): ToolResult { + return { content: [{ type: "text" as const, text }] }; +} + /** Wraps a tool handler so that thrown errors are converted to MCP error responses. */ export function withErrorHandling<P>( handler: (params: P) => ToolResult | Promise<ToolResult>, diff --git a/src/mcp/format.ts b/src/mcp/format.ts new file mode 100644 index 00000000..2098c686 --- /dev/null +++ b/src/mcp/format.ts @@ -0,0 +1,382 @@ +/** + * Compact text output for MCP tools. Every line that names a document, chunk or link carries + * its ID, so the assistant can pass it straight to the next tool. + */ +import type { IngestResult } from "../core/ingest.js"; +import type { DocumentLinks } from "../core/links.js"; +import type { DocumentView } from "../core/document-view.js"; +import type { DocumentSummary, ListResult } from "../core/operations/index.js"; +import type { Overview } from "../core/overview.js"; +import type { InstallResult, InstalledPack } from "../core/packs.js"; +import type { RegistryPack } from "../registry/types.js"; +import type { AnswerResult, RagSource } from "../core/rag.js"; +import type { ReindexResult } from "../core/reindex.js"; +import type { Rating } from "../core/ratings.js"; +import type { SearchResult } from "../core/search.js"; +import type { Task } from "../core/tasks.js"; +import type { ConnectorSyncSummary } from "../connectors/registry.js"; +import { describeEmbeddingIdentity } from "../db/index-meta.js"; + +/** Join the non-empty parts with " · ". */ +function joinParts(parts: Array<string | null | undefined | false>): string { + return parts.filter((p): p is string => typeof p === "string" && p !== "").join(" · "); +} + +/** "library v1.2" / "library" / "". */ +function libraryLabel(library: string | null, version: string | null): string { + if (!library) return ""; + return version ? `${library} v${version}` : library; +} + +/** "Showing 11-20 of 42" style header for a page of a list result. */ +function pageHeader(what: string, page: ListResult<unknown>): string { + if (page.items.length === 0) { + return page.offset > 0 ? `No more ${what} (total ${page.total}).` : `No ${what} found.`; + } + const first = page.offset + 1; + const last = page.offset + page.items.length; + const more = last < page.total ? ` — next page: offset ${last}` : ""; + const title = what.charAt(0).toUpperCase() + what.slice(1); + return `${title} ${first}-${last} of ${page.total}${more}`; +} + +function formatContext(label: string, chunks: SearchResult["contextBefore"]): string { + if (!chunks || chunks.length === 0) return ""; + return `[context ${label}]\n${chunks.map((c) => c.content).join("\n\n")}\n`; +} + +function formatSearchResult(r: SearchResult, n: number): string { + const header = joinParts([ + `[${n}] ${r.title}`, + `score ${r.score.toFixed(2)}`, + r.avgRating ? `rating ${r.avgRating.toFixed(1)}/5` : null, + ]); + const meta = joinParts([ + `documentId: ${r.documentId}`, + `chunkId: ${r.chunkId}`, + libraryLabel(r.library, r.version), + r.url, + ]); + return ( + `${header}\n${meta}\n` + + formatContext("before", r.contextBefore) + + `${r.content}\n` + + formatContext("after", r.contextAfter) + ); +} + +/** Output of `search`. */ +export function formatSearchResults(page: ListResult<SearchResult>): string { + const header = pageHeader("results", page); + if (page.items.length === 0) return header; + const body = page.items.map((r, i) => formatSearchResult(r, page.offset + i + 1)); + return `${header}\n\n${body.join("\n---\n")}`; +} + +function formatSources(sources: RagSource[]): string { + if (sources.length === 0) return ""; + const lines = sources.map( + (s) => `- ${s.title} (documentId: ${s.documentId}, score ${s.score.toFixed(2)})`, + ); + return `\n\nSources:\n${lines.join("\n")}`; +} + +/** Output of `ask`: the LLM answer, or (passthrough) the context to answer from. */ +export function formatAnswer(result: AnswerResult): string { + if (result.mode === "context") { + return ( + "Passthrough mode: no LLM was called. Answer the question yourself from this context " + + "and cite the documentIds.\n\n" + + result.contextPrompt + + formatSources(result.sources) + ); + } + const tokens = result.tokensUsed == null ? "" : `, ${result.tokensUsed} tokens`; + return `${result.answer}${formatSources(result.sources)}\n\n(model: ${result.model}${tokens})`; +} + +function formatLinks({ outgoing, incoming }: DocumentLinks): string[] { + const label = (l: { label: string | null }): string => (l.label ? ` "${l.label}"` : ""); + return [ + ...outgoing.map( + (l) => + ` -> ${l.linkType} ${l.targetTitle} (documentId: ${l.targetId}, linkId: ${l.id})${label(l)}`, + ), + ...incoming.map( + (l) => + ` <- ${l.linkType} ${l.sourceTitle} (documentId: ${l.sourceId}, linkId: ${l.id})${label(l)}`, + ), + ]; +} + +/** Header lines of a document view: IDs, metadata, tags, rating, links. */ +function formatDocumentHeader(view: DocumentView): string[] { + const d = view.document; + const { averageRating, totalRatings, corrections } = view.ratings; + const lines = [ + `# ${d.title}`, + joinParts([ + `documentId: ${d.documentId}`, + d.sourceType, + libraryLabel(d.library, d.version), + d.topicId ? `topic: ${d.topicId}` : null, + ]), + ]; + if (d.url) lines.push(`url: ${d.url}`); + if (view.tags.length > 0) lines.push(`tags: ${view.tags.join(", ")}`); + if (totalRatings > 0) { + const fixes = corrections > 0 ? `, ${corrections} suggested corrections` : ""; + lines.push(`rating: ${averageRating.toFixed(1)}/5 (${totalRatings} ratings${fixes})`); + } + const links = formatLinks(view.links); + if (links.length > 0) lines.push("links:", ...links); + lines.push(`updated: ${d.updatedAt}`); + return lines; +} + +/** Output of `get-document`: metadata, then the requested slice of the content. */ +export function formatDocumentView(view: DocumentView): string { + return `${formatDocumentHeader(view).join("\n")}${pageRange(view)}\n\n${view.content}`; +} + +/** "[characters 0-500 of 1200; next page: offset 500]" for a paged view, else "". */ +function pageRange(view: DocumentView): string { + const paged = view.offset > 0 || view.nextOffset !== null; + if (!paged) return ""; + const end = view.offset + view.content.length; + const next = view.nextOffset === null ? "]" : `; next page: offset ${view.nextOffset}]`; + return `\n\n[characters ${view.offset}-${end} of ${view.contentLength}${next}`; +} + +/** Output of `update-document`: the document's new metadata (no content). */ +export function formatDocumentUpdated(view: DocumentView): string { + return `Updated.\n${formatDocumentHeader(view).join("\n")}`; +} + +/** Output of `list-documents`. */ +export function formatDocumentList(page: ListResult<DocumentSummary>): string { + const header = pageHeader("documents", page); + const lines = page.items.map( + (d) => + `- ${d.title} (documentId: ${d.documentId}) ` + + joinParts([d.sourceType, libraryLabel(d.library, d.version), d.url]), + ); + return [header, ...lines].join("\n"); +} + +/** Output of `overview`. */ +export function formatOverview(o: Overview): string { + const { stats, health, index } = o; + const lines = [ + `Documents: ${stats.totalDocuments} · chunks: ${stats.totalChunks} · topics: ${stats.totalTopics}`, + `Health: database ${health.database} · keyword index ${health.fts} · vector search ${ + health.vectorSearch ? "ok" : "unavailable" + }`, + `Index embeddings: ${describeEmbeddingIdentity(index.stored)}` + + (index.configured ? ` · configured: ${describeEmbeddingIdentity(index.configured)}` : ""), + ]; + if (o.topics.length > 0) { + lines.push( + "Topics:", + ...o.topics.map((t) => `- ${t.name} (topic: ${t.id}, ${t.documentCount} docs)`), + ); + } + if (o.packs.length > 0) { + lines.push("Packs:", ...o.packs.map(formatInstalledPack)); + } + return lines.join("\n"); +} + +/** Output of `submit-document`. */ +export function formatIngestResult(result: IngestResult): string { + const lines: string[] = []; + if (result.planned) { + lines.push( + `Dry run: ${result.planned.length} source(s) would be added.`, + ...result.planned.map((p) => `- ${p}`), + ); + } else { + lines.push(`Added ${result.documents.length} document(s).`); + } + lines.push( + ...result.documents.map( + (d) => + `- ${d.title} (documentId: ${d.documentId}, ${d.chunkCount} chunks)` + + (d.source ? ` ${d.source}` : ""), + ), + ); + if (result.crawl) { + const { pagesCrawled, pagesSkipped, abortReason } = result.crawl; + lines.push( + `Crawled ${pagesCrawled} page(s), skipped ${pagesSkipped}` + + (abortReason ? `; stopped early: ${abortReason}` : ""), + ); + } + if (result.skipped.length > 0) { + lines.push( + `Skipped ${result.skipped.length}:`, + ...result.skipped.map((s) => `- ${s.source}: ${s.reason}`), + ); + } + if (result.errors.length > 0) { + lines.push( + `Errors ${result.errors.length}:`, + ...result.errors.map((e) => `- ${e.source}: ${e.error}`), + ); + } + return lines.join("\n"); +} + +/** Output of `delete-document`. */ +export function formatDocumentDeleted(result: { documentId: string }): string { + return `Deleted document ${result.documentId}.`; +} + +/** Output of `rate-document`. */ +export function formatRating(r: Rating): string { + const chunk = r.chunkId ? `, chunkId: ${r.chunkId}` : ""; + return joinParts([ + `Rated ${r.rating}/5 (documentId: ${r.documentId}${chunk})`, + r.feedback ? `feedback saved` : null, + r.suggestedCorrection ? `correction saved` : null, + ]); +} + +/** Output of `link-documents` with action "create". */ +export function formatLinkCreated(link: { + linkId: string; + sourceId: string; + targetId: string; + linkType: string; + label: string | null; +}): string { + const label = link.label ? ` "${link.label}"` : ""; + return ( + `Linked ${link.sourceId} -> ${link.targetId} (${link.linkType})${label}. ` + + `linkId: ${link.linkId}` + ); +} + +/** Output of `link-documents` with action "delete". */ +export function formatLinkDeleted(result: { linkId: string }): string { + return `Deleted link ${result.linkId}.`; +} + +/** Output when a tool started a background task. */ +export function formatTaskStarted(task: Task): string { + return ( + `Started background task. taskId: ${task.id}\n` + + `Poll with task {"action": "status", "taskId": "${task.id}"}.` + ); +} + +/** Formats an operation's JSON result for a task, or returns undefined to show it raw. */ +export type TaskResultFormatter = (operation: string, result: unknown) => string | undefined; + +function formatTaskResult(task: Task, formatResult?: TaskResultFormatter): string { + if (task.result === undefined) return ""; + let text: string | undefined; + if (task.operation !== undefined && formatResult) { + try { + text = formatResult(task.operation, JSON.parse(task.result) as unknown); + } catch { + text = undefined; + } + } + return `\nresult:\n${text ?? task.result}`; +} + +function taskLine(task: Task): string { + const progress = task.progress ? ` ${task.progress.current}/${task.progress.total || "?"}` : ""; + return joinParts([ + `taskId: ${task.id}`, + task.operation ?? task.type, + `${task.status}${progress}`, + `created ${task.createdAt.toISOString()}`, + ]); +} + +/** Output of `task` with action "status". */ +export function formatTask(task: Task, formatResult?: TaskResultFormatter): string { + const error = task.error ? `\nerror: ${task.error}` : ""; + return `${taskLine(task)}${error}${formatTaskResult(task, formatResult)}`; +} + +/** Output of `task` with action "cancel". */ +export function formatTaskCancel(result: { + taskId: string; + cancelRequested: boolean; + status: string; +}): string { + return result.cancelRequested + ? `Cancellation requested for task ${result.taskId} (status: ${result.status}).` + : `Task ${result.taskId} is already ${result.status}; nothing to cancel.`; +} + +/** Output of `task` with action "list". */ +export function formatTaskList(result: { items: Task[] }): string { + if (result.items.length === 0) return "No tasks in the last hour."; + return result.items.map((t) => `- ${taskLine(t)}`).join("\n"); +} + +/** Output of `sync`. */ +export function formatSyncResult(result: { + items: Array< + | { name: string; type: string; status: "completed"; summary: ConnectorSyncSummary } + | { name: string; status: "failed"; error: string } + >; +}): string { + if (result.items.length === 0) return "No saved connections to sync."; + return result.items + .map((item) => { + if (item.status === "failed") return `- ${item.name}: failed: ${item.error}`; + const { added, updated, deleted, errors } = item.summary; + const errorLines = errors.map((e) => ` ${e}`); + return [ + `- ${item.name} (${item.type}): ${added} added, ${updated} updated, ${deleted} deleted` + + (errors.length > 0 ? `, ${errors.length} errors:` : ""), + ...errorLines, + ].join("\n"); + }) + .join("\n"); +} + +function formatInstalledPack(p: InstalledPack): string { + return `- ${p.name} v${p.version} (${p.docCount} docs, installed ${p.installedAt})`; +} + +/** Output of `install-pack`. */ +export function formatInstallPack(result: InstallResult): string { + if (result.alreadyInstalled) return `Pack ${result.packName} is already installed.`; + const errors = result.errors > 0 ? `, ${result.errors} failed` : ""; + return `Installed pack ${result.packName}: ${result.documentsInstalled} documents${errors}.`; +} + +/** Output of `list-packs`. */ +export function formatPackList(result: { + items: Array<InstalledPack | RegistryPack>; + warnings?: string[] | undefined; +}): string { + const lines = result.items.map((p) => + "installedAt" in p + ? formatInstalledPack(p) + : `- ${p.name} v${p.latestVersion} (registry ${p.registry}): ${p.description}`, + ); + if (lines.length === 0) lines.push("No packs found."); + for (const warning of result.warnings ?? []) lines.push(`Warning: ${warning}`); + return lines.join("\n"); +} + +/** Output of `reindex-documents`. */ +export function formatReindex(result: ReindexResult & { rebuiltIndex?: string }): string { + const lines = [ + `Re-embedded ${result.completed} of ${result.total} chunks` + + (result.failed > 0 ? `; ${result.failed} failed` : "") + + ".", + ]; + if (result.rebuiltIndex) lines.push(`Rebuilt vector index: ${result.rebuiltIndex}`); + if (result.failedChunkIds.length > 0) { + lines.push(`Failed chunkIds: ${result.failedChunkIds.join(", ")}`); + } + return lines.join("\n"); +} diff --git a/src/mcp/main.ts b/src/mcp/main.ts new file mode 100644 index 00000000..35589c3b --- /dev/null +++ b/src/mcp/main.ts @@ -0,0 +1,15 @@ +/** + * Executable entry point: `node dist/mcp/main.js` serves libscope over MCP stdio. + * Everything else lives in server.ts, which has no side effects on import. + */ +import { runStdioServer } from "./server.js"; + +try { + await runStdioServer(); +} catch (err: unknown) { + console.error( + "libscope MCP server failed to start:", + err instanceof Error ? err.message : String(err), + ); + process.exit(1); +} diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 9071f2f9..b44a9c5d 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -1,1650 +1,403 @@ +/** + * libscope MCP server. Every tool is generated from an operation in src/core/operations: + * the tool's input schema is the operation's zod schema, and the handler runs the operation + * and formats its result as compact text. + * + * Importing this module starts nothing. `createMcpServer()` builds a server; `runStdioServer()` + * connects one to stdin/stdout (see main.ts for the executable entry point). + */ +import { createRequire } from "node:module"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import type { ToolAnnotations } from "@modelcontextprotocol/sdk/types.js"; import { z } from "zod"; -import { loadConfig } from "../config.js"; -import { getDatabase, runMigrations, createVectorTable } from "../db/index.js"; -import { getActiveWorkspace, getWorkspacePath } from "../core/workspace.js"; -import { createEmbeddingProvider } from "../providers/index.js"; -import { searchDocuments, getRelatedChunks } from "../core/search.js"; +import { loadConfig, type McpToolsetSetting } from "../config.js"; +import { bootstrap, type BootstrapOptions, type Bootstrapped } from "../core/bootstrap.js"; import { - askQuestion, - createLlmProvider, - getContextForQuestion, - isPassthroughMode, - type LlmProvider, -} from "../core/rag.js"; -import { getDocument, listDocuments, deleteDocument, updateDocument } from "../core/documents.js"; -import { rateDocument, getDocumentRatings } from "../core/ratings.js"; -import { indexDocument } from "../core/indexing.js"; -import { listTopics } from "../core/topics.js"; -import { createLink, getDocumentLinks, deleteLink } from "../core/links.js"; -import type { LinkType } from "../core/links.js"; -import { - createSavedSearch, - listSavedSearches, - runSavedSearch, - deleteSavedSearch, -} from "../core/saved-searches.js"; -import { createWebhook, listWebhooks, deleteWebhook, redactWebhook } from "../core/webhooks.js"; -import type { WebhookEvent } from "../core/webhooks.js"; -import { suggestTags } from "../core/tags.js"; -import { fetchAndConvert } from "../core/url-fetcher.js"; -import { spiderUrl } from "../core/spider.js"; -import type { SpiderOptions } from "../core/spider.js"; -import { initLogger, getLogger } from "../logger.js"; -import { ConfigError, ValidationError } from "../errors.js"; -import { errorResponse, withErrorHandling } from "./errors.js"; -export { errorResponse, withErrorHandling, type ToolResult } from "./errors.js"; -import { taskRegistry } from "./tasks.js"; -import type { TaskType } from "./tasks.js"; - -/** Build SpiderOptions from submit-document params. */ -function buildSpiderOptions( - params: { - maxPages?: number | undefined; - maxDepth?: number | undefined; - sameDomain?: boolean | undefined; - pathPrefix?: string | undefined; - excludePatterns?: string[] | undefined; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): SpiderOptions { - const opts: SpiderOptions = { fetchOptions }; - if (params.maxPages !== undefined) opts.maxPages = params.maxPages; - if (params.maxDepth !== undefined) opts.maxDepth = params.maxDepth; - if (params.sameDomain !== undefined) opts.sameDomain = params.sameDomain; - if (params.pathPrefix !== undefined) opts.pathPrefix = params.pathPrefix; - if (params.excludePatterns !== undefined) opts.excludePatterns = params.excludePatterns; - return opts; + addOperation, + askOperation, + cancelTaskOperation, + createOperationContext, + deleteDocumentOperation, + getDocumentOperation, + getTaskOperation, + installPackOperation, + linkDocumentsOperation, + listDocumentsOperation, + listPacksOperation, + listTasksOperation, + overviewOperation, + rateDocumentOperation, + reindexOperation, + runOperation, + searchOperation, + startOperationTask, + syncOperation, + unlinkDocumentsOperation, + updateDocumentOperation, + type Operation, + type OperationAnnotations, + type OperationContext, +} from "../core/operations/index.js"; +import { initLogger, type LogLevel } from "../logger.js"; +import { textResult, withErrorHandling } from "./errors.js"; +import * as fmt from "./format.js"; + +export { errorResponse, textResult, withErrorHandling, type ToolResult } from "./errors.js"; + +/** Optional toolsets, enabled with the `mcp.toolsets` config key. */ +export const MCP_TOOLSETS = ["admin"] as const; +export type McpToolset = (typeof MCP_TOOLSETS)[number]; + +/** Formats an operation result as tool output text. */ +type Formatter<O> = (result: O) => string; + +export interface OperationToolOptions<O> { + /** Tool name (default: the operation name). */ + name?: string | undefined; + /** Tool description (default: the operation summary). */ + description?: string | undefined; + /** Operation input fields the tool does not offer. */ + omit?: readonly string[] | undefined; + format: Formatter<O>; } -/** Handle spider mode for submit-document. */ -async function handleSpiderSubmit( - db: import("better-sqlite3").Database, - provider: import("../providers/embedding.js").EmbeddingProvider, - params: { - url?: string | undefined; - library?: string | undefined; - version?: string | undefined; - topic?: string | undefined; - sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; - maxPages?: number | undefined; - maxDepth?: number | undefined; - sameDomain?: boolean | undefined; - pathPrefix?: string | undefined; - excludePatterns?: string[] | undefined; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): Promise<{ content: Array<{ type: "text"; text: string }> }> { - const { url, library, version, topic } = params; - if (!url) { - throw new ValidationError("Field 'url' is required when spider is true"); - } - - const spiderOptions = buildSpiderOptions(params, fetchOptions); - const indexed: Array<{ id: string; title: string }> = []; - const errors: Array<{ url: string; error: string }> = []; - const sourceType = params.sourceType ?? (library ? "library" : "manual"); - - const gen = spiderUrl(url, spiderOptions); - let result = await gen.next(); - while (!result.done) { - const page = result.value; - try { - const doc = await indexDocument(db, provider, { - title: page.title, - content: page.content, - sourceType, - library, - version, - topicId: topic, - url: page.url, - submittedBy: "model", - }); - indexed.push({ id: doc.id, title: page.title }); - } catch (err) { - errors.push({ url: page.url, error: err instanceof Error ? err.message : String(err) }); - } - result = await gen.next(); - } - const stats = result.value; - - const summary = [ - `Spider complete.`, - `Pages indexed: ${indexed.length}`, - `Pages crawled: ${stats?.pagesCrawled ?? indexed.length}`, - `Pages skipped: ${stats?.pagesSkipped ?? 0}`, - errors.length > 0 ? `Errors: ${errors.length}` : null, - stats?.abortReason ? `Stopped early: ${stats.abortReason}` : null, - ] - .filter(Boolean) - .join("\n"); - - return { content: [{ type: "text" as const, text: summary }] }; -} - -/** Handle single-document submission for submit-document. */ -async function handleSingleDocSubmit( - db: import("better-sqlite3").Database, - provider: import("../providers/embedding.js").EmbeddingProvider, - params: { - title?: string | undefined; - content?: string | undefined; - url?: string | undefined; - library?: string | undefined; - version?: string | undefined; - topic?: string | undefined; - sourceType?: "library" | "topic" | "manual" | "model-generated" | undefined; - }, - fetchOptions: { allowPrivateUrls: boolean; allowSelfSignedCerts: boolean }, -): Promise<{ content: Array<{ type: "text"; text: string }> }> { - let { title, content } = params; - const { url, library, version, topic } = params; - - if (url && !content) { - const fetched = await fetchAndConvert(url, fetchOptions); - content = fetched.content; - title ??= fetched.title; - } - - if (!title) { - throw new ValidationError("A title is required when not providing a URL"); - } - if (!content) { - throw new ValidationError("Either content or a URL must be provided"); - } - - const sourceType = params.sourceType ?? (library ? "library" : "manual"); - - const result = await indexDocument(db, provider, { - title, - content, - sourceType, - library, - version, - topicId: topic, - url, - submittedBy: "model", - }); +const asyncField = z + .boolean() + .default(false) + .describe( + 'Run in the background and return a taskId at once; poll with task {"action": "status"}', + ); +/** MCP tool hints from operation annotations (several for a multi-action tool). */ +function toolAnnotations(list: OperationAnnotations[]): ToolAnnotations { + if (list.every((a) => a.readOnly === true)) return { readOnlyHint: true }; return { - content: [ - { - type: "text" as const, - text: - `Document indexed successfully.\n` + - `Title: ${title}\n` + - `ID: ${result.id}\n` + - `Chunks: ${result.chunkCount}` + - (url ? `\nSource: ${url}` : ""), - }, - ], + readOnlyHint: false, + destructiveHint: list.some((a) => a.destructive === true), + idempotentHint: list.every((a) => a.readOnly === true || a.idempotent === true), }; } -/** Fire-and-forget helper: creates a task, runs `work` in background, returns task ID response. */ -function startAsyncTask( - type: TaskType, - work: ( - signal: AbortSignal, - onProgress: (current: number, total: number) => void, - ) => Promise<string>, -): { content: Array<{ type: "text"; text: string }> } { - const { task, signal } = taskRegistry.create(type); - taskRegistry.update(task.id, { status: "running", startedAt: new Date() }); - const onProgress = (current: number, total: number): void => { - taskRegistry.update(task.id, { progress: { current, total } }); - }; - void work(signal, onProgress).then( - (result) => { - if (signal.aborted) { - taskRegistry.update(task.id, { status: "cancelled", completedAt: new Date() }); - } else { - taskRegistry.update(task.id, { status: "completed", completedAt: new Date(), result }); - } - }, - (err: unknown) => { - if (signal.aborted) { - taskRegistry.update(task.id, { status: "cancelled", completedAt: new Date() }); - } else { - taskRegistry.update(task.id, { - status: "failed", - completedAt: new Date(), - error: err instanceof Error ? err.message : String(err), - }); - } - }, - ); - return { - content: [ - { type: "text" as const, text: `Task queued. ID: ${task.id}\nUse get-task to check status.` }, - ], - }; +function shapeOf(op: Operation): Record<string, z.ZodType> { + return op.input.shape as Record<string, z.ZodType>; } -// Start the server -async function main(): Promise<void> { - let config; - try { - config = loadConfig(); - } catch (err) { - console.error("Failed to load configuration:", err instanceof Error ? err.message : err); - process.exit(1); - } - - initLogger(config.logging.level); - - let db; - try { - db = getDatabase(getWorkspacePath(getActiveWorkspace())); - runMigrations(db); - } catch (err) { - console.error("Failed to initialize database:", err instanceof Error ? err.message : err); - process.exit(1); - } - - let provider; - let llmProvider: LlmProvider | undefined; - try { - provider = createEmbeddingProvider(config); - createVectorTable(db, provider.dimensions); - } catch (err) { - console.error( - "Failed to initialize embedding provider:", - err instanceof Error ? err.message : err, - ); - db.close(); - process.exit(1); - } - - try { - llmProvider = createLlmProvider(config); - } catch (err) { - getLogger().warn({ err }, "LLM provider unavailable — ask-question tool will not work"); - } - - process.on("SIGINT", () => { - db.close(); - process.exit(0); - }); - process.on("SIGTERM", () => { - db.close(); - process.exit(0); - }); - - const server = new McpServer({ - name: "libscope", - version: "0.1.0", - }); - - // Tool: search-docs - server.tool( - "search-docs", - "Semantic search across all indexed documentation, library docs, and topics", +/** + * Register `op` as an MCP tool. The input schema is the operation's, minus `omit`; a + * long-running operation also gets an `async` flag that runs it as a background task. + */ +export function registerOperationTool<S extends z.ZodObject, O>( + server: McpServer, + op: Operation<S, O>, + ctx: OperationContext, + options: OperationToolOptions<O>, +): string { + const omit = new Set(options.omit ?? []); + const shape = Object.fromEntries( + Object.entries(shapeOf(op as unknown as Operation)).filter(([key]) => !omit.has(key)), + ); + const background = op.annotations?.longRunning === true; + if (background) shape["async"] = asyncField; + const name = options.name ?? op.name; + server.registerTool( + name, { - query: z.string().describe("The search query"), - topic: z.string().optional().describe("Filter by topic ID"), - library: z.string().optional().describe("Filter by library name"), - version: z.string().optional().describe("Filter by library version"), - source: z - .string() - .optional() - .describe("Filter by source type (e.g., 'library', 'topic', 'manual', 'model-generated')"), - minRating: z.number().min(1).max(5).optional().describe("Minimum average rating filter"), - offset: z.number().min(0).optional().describe("Offset for pagination (default: 0)"), - limit: z - .number() - .min(1) - .max(50) - .optional() - .describe("Maximum results to return (default: 10)"), - maxChunksPerDocument: z - .number() - .min(0) - .max(50) - .optional() - .describe( - "Maximum chunks per document in results (default: no limit, set to 2 for diversity)", - ), - contextChunks: z - .number() - .min(0) - .max(2) - .optional() - .describe( - "Number of neighboring chunks to include before/after each result for context (0-2, default: 0)", - ), - }, - withErrorHandling(async (params) => { - const { results, totalCount } = await searchDocuments(db, provider, { - query: params.query, - topic: params.topic, - library: params.library, - version: params.version, - source: params.source, - minRating: params.minRating, - limit: params.limit, - offset: params.offset, - maxChunksPerDocument: params.maxChunksPerDocument, - contextChunks: params.contextChunks, - }); - - if (results.length === 0) { - return { - content: [{ type: "text" as const, text: "No documents found matching your query." }], - }; + description: options.description ?? op.summary, + inputSchema: z.object(shape), + annotations: toolAnnotations([op.annotations ?? {}]), + }, + withErrorHandling(async (args: Record<string, unknown>) => { + const { async: runInBackground, ...input } = args; + if (background && runInBackground === true) { + return textResult(fmt.formatTaskStarted(startOperationTask(op, ctx, input))); } - - const text = - `**Total results: ${totalCount}**\n\n` + - results - .map((r, i) => { - const libraryVersion = r.version ? ` v${r.version}` : ""; - let entry = - `## Result ${i + 1}: ${r.title} (score: ${r.score.toFixed(2)})\n` + - (r.library ? `**Library:** ${r.library}${libraryVersion}\n` : "") + - (r.url ? `**Source:** ${r.url}\n` : "") + - (r.avgRating ? `**Rating:** ${r.avgRating.toFixed(1)}/5\n` : ""); - - if (r.contextBefore && r.contextBefore.length > 0) { - entry += `\n**Context (before):**\n${r.contextBefore.map((c) => c.content).join("\n\n")}\n`; - } - - entry += `\n${r.content}\n`; - - if (r.contextAfter && r.contextAfter.length > 0) { - entry += `\n**Context (after):**\n${r.contextAfter.map((c) => c.content).join("\n\n")}\n`; - } - - return entry; - }) - .join("\n---\n\n"); - - return { content: [{ type: "text" as const, text }] }; - }), - ); - - // Tool: get-related - server.tool( - "get-related", - "Find chunks semantically similar to a given chunk (more-like-this). Returns related content seeded from an existing chunk's stored embedding without requiring a text query.", - { - chunkId: z.string().describe("ID of the source chunk to find related content for"), - limit: z - .number() - .min(1) - .max(50) - .optional() - .describe("Number of results to return (default 10)"), - topic: z.string().optional().describe("Filter results to a specific topic"), - library: z.string().optional().describe("Filter results to a specific library"), - tags: z.array(z.string()).optional().describe("Filter results to documents with these tags"), - minScore: z - .number() - .min(0) - .max(1) - .optional() - .describe("Minimum similarity score threshold (0-1)"), - includeLinkedDocuments: z - .boolean() - .optional() - .describe("Also include explicitly linked documents even if below similarity threshold"), - }, - withErrorHandling( - ({ chunkId, limit, topic, library, tags, minScore, includeLinkedDocuments }) => { - const result = getRelatedChunks(db, { - chunkId, - ...(limit !== undefined && { limit }), - ...(topic !== undefined && { topic }), - ...(library !== undefined && { library }), - ...(tags !== undefined && { tags }), - ...(minScore !== undefined && { minScore }), - ...(includeLinkedDocuments !== undefined && { includeLinkedDocuments }), - }); - return { - content: [ - { - type: "text" as const, - text: JSON.stringify(result, null, 2), - }, - ], - }; - }, - ), - ); - - // Tool: get-document - server.tool( - "get-document", - "Retrieve a specific document by its ID", - { - documentId: z.string().describe("The document ID"), - }, - withErrorHandling((params) => { - const doc = getDocument(db, params.documentId); - const ratings = getDocumentRatings(db, params.documentId); - - const docVersion = doc.version ? ` v${doc.version}` : ""; - const text = - `# ${doc.title}\n\n` + - `**Type:** ${doc.sourceType}\n` + - (doc.library ? `**Library:** ${doc.library}${docVersion}\n` : "") + - (doc.url ? `**Source:** ${doc.url}\n` : "") + - `**Rating:** ${ratings.averageRating.toFixed(1)}/5 (${ratings.totalRatings} ratings)\n\n` + - doc.content; - - return { content: [{ type: "text" as const, text }] }; - }), - ); - - // Tool: delete-document - server.tool( - "delete-document", - "Delete a document from the knowledge base by its ID", - { - documentId: z.string().describe("The document ID to delete"), - }, - withErrorHandling((params) => { - deleteDocument(db, params.documentId); - - return { - content: [ - { - type: "text" as const, - text: `Document ${params.documentId} has been deleted successfully.`, - }, - ], - }; - }), - ); - - // Tool: update-document - server.tool( - "update-document", - "Update an existing document's title, content, or metadata", - { - documentId: z.string().describe("The document ID to update"), - title: z.string().optional().describe("New title"), - content: z.string().optional().describe("New content (will re-chunk and re-index)"), - library: z.string().nullable().optional().describe("New library name (null to clear)"), - version: z.string().nullable().optional().describe("New version (null to clear)"), - url: z.string().nullable().optional().describe("New URL (null to clear)"), - topicId: z.string().nullable().optional().describe("New topic ID (null to clear)"), - }, - withErrorHandling(async (params) => { - const metadata: Record<string, string | null | undefined> = {}; - if (params.library !== undefined) metadata.library = params.library; - if (params.version !== undefined) metadata.version = params.version; - if (params.url !== undefined) metadata.url = params.url; - if (params.topicId !== undefined) metadata.topicId = params.topicId; - - const doc = await updateDocument(db, provider, params.documentId, { - title: params.title, - content: params.content, - metadata: - Object.keys(metadata).length > 0 - ? (metadata as { - library?: string | null; - version?: string | null; - url?: string | null; - topicId?: string | null; - }) - : undefined, - }); - return { - content: [{ type: "text" as const, text: `Document updated: ${doc.title} (${doc.id})` }], - }; + return textResult(options.format(await runOperation(op, ctx, input))); }), ); + return name; +} - // Tool: rate-document - server.tool( - "rate-document", - "Rate a document or suggest corrections. Use this when documentation appears outdated, incorrect, or particularly helpful.", - { - documentId: z.string().describe("The document ID to rate"), - chunkId: z.string().optional().describe("Optional specific chunk ID to rate"), - rating: z.number().min(1).max(5).describe("Rating from 1 (poor) to 5 (excellent)"), - feedback: z.string().optional().describe("Text feedback about the document"), - suggestedCorrection: z - .string() - .optional() - .describe("Suggested replacement content if the doc is wrong"), - }, - withErrorHandling((params) => { - const result = rateDocument(db, { - documentId: params.documentId, - chunkId: params.chunkId, - rating: params.rating, - feedback: params.feedback, - suggestedCorrection: params.suggestedCorrection, - ratedBy: "model", - }); - - return { - content: [ - { - type: "text" as const, - text: - `Rating submitted: ${result.rating}/5 for document ${result.documentId}` + - (result.feedback ? `\nFeedback: ${result.feedback}` : "") + - (result.suggestedCorrection ? `\nCorrection suggested.` : ""), - }, - ], - }; - }), - ); - - // Tool: submit-document - server.tool( - "submit-document", - "Submit a new document for indexing into the knowledge base. You can provide content directly, or provide a URL to fetch and index automatically. Set spider=true to crawl linked pages from the URL.", - { - title: z - .string() - .optional() - .describe("Document title (auto-detected from URL if not provided)"), - content: z - .string() - .optional() - .describe("Document content in markdown (omit if providing a URL to fetch)"), - url: z - .string() - .optional() - .describe( - "URL to fetch and index. When provided, content is fetched automatically. Title is auto-detected if not specified.", - ), - sourceType: z - .enum(["library", "topic", "manual", "model-generated"]) - .optional() - .describe("Type of document (default: 'manual', or 'library' if library name is given)"), - topic: z.string().optional().describe("Topic ID to categorize under"), - library: z.string().optional().describe("Library name (for library docs)"), - version: z.string().optional().describe("Library version"), - spider: z - .boolean() - .optional() - .describe("When true, crawl pages linked from the URL. Requires 'url'. Default: false."), - maxPages: z - .number() - .int() - .positive() - .optional() - .describe("Maximum pages to index during a spider run (default: 25, hard cap: 200)."), - maxDepth: z - .number() - .int() - .min(0) - .optional() - .describe( - "Maximum link-hop depth from the seed URL (default: 2, hard cap: 5). 0 = seed only.", - ), - sameDomain: z - .boolean() - .optional() - .describe("Only follow links on the same domain as the seed URL (default: true)."), - pathPrefix: z - .string() - .optional() - .describe("Only follow links whose path starts with this prefix (e.g. '/docs/')."), - excludePatterns: z - .array(z.string()) - .optional() - .describe("Glob patterns for URLs to skip (e.g. ['*/changelog*', '*/api/v1/*'])."), - async: z - .boolean() - .optional() - .describe( - "When true, start indexing in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const fetchOptions = { - allowPrivateUrls: config.indexing.allowPrivateUrls, - allowSelfSignedCerts: config.indexing.allowSelfSignedCerts, - }; - - if (params.async) { - return startAsyncTask("index_document", async () => { - if (params.spider) { - const r = await handleSpiderSubmit(db, provider, params, fetchOptions); - return r.content[0]?.text ?? "Done"; - } - const r = await handleSingleDocSubmit(db, provider, params, fetchOptions); - return r.content[0]?.text ?? "Done"; - }); - } - - if (params.spider) { - return handleSpiderSubmit(db, provider, params, fetchOptions); - } - - return handleSingleDocSubmit(db, provider, params, fetchOptions); - }), - ); - - // Tool: list-topics - server.tool( - "list-topics", - "List available documentation topics", - { - parentId: z.string().optional().describe("Filter by parent topic ID for subtopics"), - }, - withErrorHandling((params) => { - const topics = listTopics(db, params.parentId); - - if (topics.length === 0) { - return { - content: [{ type: "text" as const, text: "No topics found." }], - }; - } - - const text = topics - .map((t) => { - const topicDesc = t.description ? `: ${t.description}` : ""; - return `- **${t.name}** (\`${t.id}\`)${topicDesc}`; - }) - .join("\n"); - - return { content: [{ type: "text" as const, text: `## Topics\n\n${text}` }] }; - }), - ); - - // Tool: health-check - server.tool( - "health-check", - "Check the health of the LibScope server, including database connectivity, document and chunk counts, and FTS5 index status", - {}, - () => { - try { - const health: Record<string, unknown> = {}; - - // Check database connectivity - try { - db.prepare("SELECT 1").get(); - health.database = "ok"; - } catch (err: unknown) { - health.database = "error"; - getLogger().warn({ err }, "Health check: database connectivity failed"); - } - - // Document count - try { - const row = db.prepare("SELECT COUNT(*) as count FROM documents").get() as { - count: number; - }; - health.documents = row.count; - } catch (err: unknown) { - health.documents = "error"; - getLogger().warn({ err }, "Health check: document count query failed"); - } - - // Chunk count - try { - const row = db.prepare("SELECT COUNT(*) as count FROM chunks").get() as { - count: number; - }; - health.chunks = row.count; - } catch (err: unknown) { - health.chunks = "error"; - getLogger().warn({ err }, "Health check: chunk count query failed"); - } - - // FTS5 index status - try { - db.prepare("SELECT COUNT(*) FROM chunks_fts").get(); - health.fts5 = "ok"; - } catch (err: unknown) { - health.fts5 = "error"; - getLogger().warn({ err }, "Health check: FTS5 index query failed"); - } - - return { - content: [{ type: "text" as const, text: JSON.stringify(health, null, 2) }], - }; - } catch (err) { - return errorResponse(err); - } - }, - ); - - // Tool: list-documents - server.tool( - "list-documents", - "List all indexed documents with optional filters", - { - library: z.string().optional().describe("Filter by library name"), - topic: z.string().optional().describe("Filter by topic ID"), - sourceType: z - .enum(["library", "topic", "manual", "model-generated"]) - .optional() - .describe("Filter by source type"), - limit: z.number().min(1).max(100).optional().describe("Maximum results (default: 50)"), - }, - withErrorHandling((params) => { - const docs = listDocuments(db, { - library: params.library, - topicId: params.topic, - sourceType: params.sourceType, - limit: params.limit, - }); - - if (docs.length === 0) { - return { content: [{ type: "text" as const, text: "No documents found." }] }; - } - - const text = docs - .map((d) => { - const docLibVersion = d.version ? ` v${d.version}` : ""; - return ( - `- **${d.title}** (\`${d.id}\`)` + - (d.library ? ` — ${d.library}${docLibVersion}` : "") + - (d.url ? ` — [source](${d.url})` : "") + - ` (${d.sourceType})` - ); - }) - .join("\n"); - - return { - content: [{ type: "text" as const, text: `## Documents (${docs.length})\n\n${text}` }], - }; - }), - ); - - // Tool: ask-question (RAG) - server.tool( - "ask-question", - "Ask a question and get an LLM-synthesized answer based on indexed documentation (RAG)", - { - question: z.string().describe("The question to answer"), - topK: z - .number() - .min(1) - .max(20) - .optional() - .describe("Number of chunks to retrieve for context (default: 5)"), - topic: z.string().optional().describe("Filter by topic ID"), - library: z.string().optional().describe("Filter by library name"), - }, - withErrorHandling(async (params) => { - if (isPassthroughMode(config)) { - const { contextPrompt, sources } = await getContextForQuestion(db, provider, { - question: params.question, - topK: params.topK, - topic: params.topic, - library: params.library, - }); - - const sourcesText = - sources.length > 0 - ? "\n\n**Sources:**\n" + - sources - .map((s) => `- ${s.title} (score: ${s.score.toFixed(2)}) [${s.documentId}]`) - .join("\n") - : ""; - - return { - content: [{ type: "text" as const, text: contextPrompt + sourcesText }], - }; - } - - if (!llmProvider) { - throw new ConfigError( - "No LLM provider configured. Set llm.provider to 'openai', 'ollama', or 'passthrough' in your config.", - ); - } - - const result = await askQuestion(db, provider, llmProvider, { - question: params.question, - topK: params.topK, - topic: params.topic, - library: params.library, - }); - - const sourcesText = - result.sources.length > 0 - ? "\n\n**Sources:**\n" + - result.sources - .map((s) => `- ${s.title} (score: ${s.score.toFixed(2)}) [${s.documentId}]`) - .join("\n") - : ""; - - const metaText = - result.tokensUsed != null - ? `\n\n_Model: ${result.model} | Tokens: ${result.tokensUsed}_` - : ""; - - return { - content: [{ type: "text" as const, text: result.answer + sourcesText + metaText }], - }; - }), - ); - - // Tool: reindex-documents - server.tool( - "reindex-documents", - "Re-embed all document chunks with the current embedding model. Use after switching embedding providers to update vectors without re-fetching content.", - { - documentIds: z - .array(z.string()) - .optional() - .describe("Only reindex chunks belonging to these document IDs"), - since: z - .string() - .optional() - .describe("Only reindex documents created on or after this ISO-8601 date"), - before: z - .string() - .optional() - .describe("Only reindex documents created on or before this ISO-8601 date"), - batchSize: z - .number() - .min(1) - .max(500) - .optional() - .describe("Chunks per embedding batch (default: 50)"), - async: z - .boolean() - .optional() - .describe( - "When true, run reindexing in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { reindex } = await import("../core/reindex.js"); - - if (params.async) { - return startAsyncTask("reindex_library", async (signal, onProgress) => { - const result = await reindex(db, provider, { - documentIds: params.documentIds, - since: params.since, - before: params.before, - batchSize: params.batchSize, - onProgress: (p) => { - if (signal.aborted) throw new Error("Task cancelled"); - onProgress(p.completed, p.total); - }, - }); - return ( - `Reindex complete.\n` + - `Total chunks: ${result.total}\n` + - `Updated: ${result.completed}\n` + - `Failed: ${result.failed}` + - (result.failedChunkIds.length > 0 - ? `\nFailed chunk IDs: ${result.failedChunkIds.join(", ")}` - : "") - ); - }); - } - - const result = await reindex(db, provider, { - documentIds: params.documentIds, - since: params.since, - before: params.before, - batchSize: params.batchSize, - }); - - const text = - `Reindex complete.\n` + - `Total chunks: ${result.total}\n` + - `Updated: ${result.completed}\n` + - `Failed: ${result.failed}` + - (result.failedChunkIds.length > 0 - ? `\nFailed chunk IDs: ${result.failedChunkIds.join(", ")}` - : ""); - - return { content: [{ type: "text" as const, text }] }; - }), - ); - - // Tool: sync-slack - server.tool( - "sync-slack", - "Sync Slack channel messages and threads into the knowledge base", - { - token: z.string().describe("Slack bot token (xoxb-...) or user token (xoxp-...)"), - channels: z - .array(z.string()) - .describe("Channel names or IDs to sync, or ['all'] for all channels"), - excludeChannels: z - .array(z.string()) - .optional() - .describe("Channel names to exclude from sync"), - threadMode: z - .enum(["aggregate", "separate"]) - .optional() - .describe( - "Thread handling: aggregate (default) combines thread into one doc, separate creates one doc per reply", - ), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { syncSlack: doSyncSlack } = await import("../connectors/slack.js"); - - const slackConfig = { - token: params.token, - channels: params.channels, - excludeChannels: params.excludeChannels, - threadMode: params.threadMode ?? ("aggregate" as const), - }; - - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await doSyncSlack(db, provider, slackConfig); - const slackErrorLines = result.errors - .map((e) => ` #${e.channel}: ${e.error}`) - .join("\n"); - const slackErrors = result.errors.length > 0 ? `\nErrors:\n${slackErrorLines}` : ""; - return ( - `Slack sync complete.\n` + - `Channels: ${result.channels}\n` + - `Messages indexed: ${result.messagesIndexed}\n` + - `Threads indexed: ${result.threadsIndexed}` + - slackErrors - ); - }); - } - - const result = await doSyncSlack(db, provider, slackConfig); - - const slackErrorLines = result.errors.map((e) => ` #${e.channel}: ${e.error}`).join("\n"); - const slackErrors = result.errors.length > 0 ? `\nErrors:\n${slackErrorLines}` : ""; - const text = - `Slack sync complete.\n` + - `Channels: ${result.channels}\n` + - `Messages indexed: ${result.messagesIndexed}\n` + - `Threads indexed: ${result.threadsIndexed}` + - slackErrors; - - return { content: [{ type: "text" as const, text }] }; - }), - ); - - // Tool: install-pack - server.tool( - "install-pack", - "Install a knowledge pack from the registry or a local file path", - { - nameOrPath: z.string().describe("Pack name (from registry) or local .json file path"), - registryUrl: z.string().optional().describe("Custom registry URL"), - async: z - .boolean() - .optional() - .describe( - "When true, run installation in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { installPack } = await import("../core/packs.js"); - - if (params.async) { - return startAsyncTask("install_pack", async (signal, onProgress) => { - const result = await installPack(db, provider, params.nameOrPath, { - registryUrl: params.registryUrl, - onProgress: (current, total) => { - if (signal.aborted) throw new Error("Task cancelled"); - onProgress(current, total); - }, - }); - return result.alreadyInstalled - ? `Pack "${result.packName}" is already installed.` - : `Pack "${result.packName}" installed successfully (${result.documentsInstalled} documents).`; - }); - } - - const result = await installPack(db, provider, params.nameOrPath, { - registryUrl: params.registryUrl, - }); - - if (result.alreadyInstalled) { - return { - content: [ - { type: "text" as const, text: `Pack "${result.packName}" is already installed.` }, - ], - }; - } - - return { - content: [ - { - type: "text" as const, - text: `Pack "${result.packName}" installed successfully (${result.documentsInstalled} documents).`, - }, - ], - }; - }), - ); - - // Tool: list-packs - server.tool( - "list-packs", - "List installed knowledge packs or available packs from the registry", - { - available: z - .boolean() - .optional() - .describe("If true, list available packs from registry instead of installed packs"), - registryUrl: z.string().optional().describe("Custom registry URL"), - }, - withErrorHandling(async (params) => { - if (params.available) { - const { listAvailablePacks } = await import("../core/packs.js"); - const packs = await listAvailablePacks(params.registryUrl); - if (packs.length === 0) { - return { - content: [{ type: "text" as const, text: "No packs available in the registry." }], - }; - } - const text = packs - .map((p) => `- **${p.name}** v${p.version} — ${p.description} (${p.docCount} docs)`) - .join("\n"); - return { content: [{ type: "text" as const, text: `## Available Packs\n\n${text}` }] }; - } - - const { listInstalledPacks } = await import("../core/packs.js"); - const packs = listInstalledPacks(db); - if (packs.length === 0) { - return { content: [{ type: "text" as const, text: "No packs installed." }] }; - } - const text = packs - .map( - (p) => - `- **${p.name}** v${p.version} — ${p.description ?? ""} (${p.docCount} docs, installed ${p.installedAt})`, - ) - .join("\n"); - return { content: [{ type: "text" as const, text: `## Installed Packs\n\n${text}` }] }; - }), - ); - - // Tool: sync-onenote - server.tool( - "sync-onenote", - "Sync OneNote notebooks via Microsoft Graph API", - { - accessToken: z.string().describe("Microsoft Graph API access token"), - notebookName: z.string().optional().describe("Specific notebook name to sync (default: all)"), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { syncOneNote } = await import("../connectors/onenote.js"); - - const oneNoteConfig = { - clientId: "", - tenantId: "common", - accessToken: params.accessToken, - notebooks: params.notebookName ? [params.notebookName] : ["all"], - excludeSections: [] as string[], - }; - - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await syncOneNote(db, provider, oneNoteConfig); - const oneNoteErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const oneNoteErrors = result.errors.length > 0 ? `\nErrors: ${oneNoteErrorLines}` : ""; - return ( - `OneNote sync complete.\n` + - `Notebooks: ${result.notebooks}\n` + - `Sections: ${result.sections}\n` + - `Pages added: ${result.pagesAdded}\n` + - `Pages updated: ${result.pagesUpdated}\n` + - `Pages deleted: ${result.pagesDeleted}` + - oneNoteErrors - ); - }); - } - - const result = await syncOneNote(db, provider, oneNoteConfig); - - const oneNoteErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const oneNoteErrors = result.errors.length > 0 ? `\nErrors: ${oneNoteErrorLines}` : ""; - const text = - `OneNote sync complete.\n` + - `Notebooks: ${result.notebooks}\n` + - `Sections: ${result.sections}\n` + - `Pages added: ${result.pagesAdded}\n` + - `Pages updated: ${result.pagesUpdated}\n` + - `Pages deleted: ${result.pagesDeleted}` + - oneNoteErrors; - - return { content: [{ type: "text" as const, text }] }; - }), - ); - - // Tool: sync-notion - server.tool( - "sync-notion", - "Sync pages and databases from a connected Notion workspace into the knowledge base", - { - token: z.string().describe("Notion integration token (secret_... or ntn_...)"), - lastSync: z - .string() - .optional() - .describe("ISO-8601 timestamp — only sync pages edited after this time"), - excludePages: z - .array(z.string()) - .optional() - .describe("List of Notion page/database IDs to exclude from sync"), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { syncNotion } = await import("../connectors/notion.js"); - - const notionConfig = { - token: params.token, - lastSync: params.lastSync, - excludePages: params.excludePages, - }; - - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await syncNotion(db, provider, notionConfig); - const notionErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const notionErrors = result.errors.length > 0 ? `\nErrors: ${notionErrorLines}` : ""; - return ( - `Notion sync complete.\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Databases indexed: ${result.databasesIndexed}` + - notionErrors - ); - }); - } - - const result = await syncNotion(db, provider, notionConfig); - - const notionErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join("; "); - const notionErrors = result.errors.length > 0 ? `\nErrors: ${notionErrorLines}` : ""; - const text = - `Notion sync complete.\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Databases indexed: ${result.databasesIndexed}` + - notionErrors; - - return { content: [{ type: "text" as const, text }] }; - }), - ); - - // Tool: sync-obsidian-vault - server.tool( - "sync-obsidian-vault", - "Sync an Obsidian vault into the knowledge base. Parses wikilinks, frontmatter, embeds, and tags with incremental sync support.", - { - vaultPath: z.string().describe("Absolute path to the Obsidian vault directory"), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { syncObsidianVault } = await import("../connectors/obsidian.js"); - - const obsidianConfig = { - vaultPath: params.vaultPath, - topicMapping: "folder" as const, - excludePatterns: [] as string[], - }; - - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await syncObsidianVault(db, provider, obsidianConfig); - const obsidianErrorLines = result.errors.map((e) => `${e.file}: ${e.error}`).join(", "); - const obsidianErrors = result.errors.length > 0 ? `\nErrors: ${obsidianErrorLines}` : ""; - return ( - `Obsidian vault sync complete.\n` + - `Added: ${result.added}\n` + - `Updated: ${result.updated}\n` + - `Deleted: ${result.deleted}` + - obsidianErrors - ); - }); - } - - const result = await syncObsidianVault(db, provider, obsidianConfig); - - const obsidianErrorLines = result.errors.map((e) => `${e.file}: ${e.error}`).join(", "); - const obsidianErrors = result.errors.length > 0 ? `\nErrors: ${obsidianErrorLines}` : ""; - const text = - `Obsidian vault sync complete.\n` + - `Added: ${result.added}\n` + - `Updated: ${result.updated}\n` + - `Deleted: ${result.deleted}` + - obsidianErrors; - - return { content: [{ type: "text" as const, text }] }; - }), - ); - - // Tool: sync-confluence - server.tool( - "sync-confluence", - "Sync Confluence spaces and pages into the knowledge base", - { - baseUrl: z.string().describe("Confluence base URL (e.g. https://acme.atlassian.net)"), - email: z.string().describe("Confluence user email"), - token: z.string().describe("API token or PAT"), - spaces: z - .array(z.string()) - .optional() - .describe("Space keys to sync, or ['all'] (default: ['all'])"), - excludeSpaces: z.array(z.string()).optional().describe("Space keys to exclude"), - async: z - .boolean() - .optional() - .describe( - "When true, run the sync in the background and return a task ID immediately. Use get-task to poll for completion.", - ), - }, - withErrorHandling(async (params) => { - const { syncConfluence } = await import("../connectors/confluence.js"); - - const confluenceConfig = { - baseUrl: params.baseUrl, - email: params.email, - token: params.token, - spaces: params.spaces ?? ["all"], - excludeSpaces: params.excludeSpaces, - }; - - if (params.async) { - return startAsyncTask("sync_connector", async () => { - const result = await syncConfluence(db, provider, confluenceConfig); - const confluenceErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join(", "); - const confluenceErrors = - result.errors.length > 0 ? `\nErrors: ${confluenceErrorLines}` : ""; - return ( - `Confluence sync complete.\n` + - `Spaces: ${result.spaces}\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Pages updated: ${result.pagesUpdated}` + - confluenceErrors - ); - }); - } - - const result = await syncConfluence(db, provider, confluenceConfig); - - const confluenceErrorLines = result.errors.map((e) => `${e.page}: ${e.error}`).join(", "); - const confluenceErrors = result.errors.length > 0 ? `\nErrors: ${confluenceErrorLines}` : ""; - const text = - `Confluence sync complete.\n` + - `Spaces: ${result.spaces}\n` + - `Pages indexed: ${result.pagesIndexed}\n` + - `Pages updated: ${result.pagesUpdated}` + - confluenceErrors; +/** One action of a multi-action tool. */ +export interface ToolAction { + op: Operation; + format: Formatter<unknown>; +} - return { content: [{ type: "text" as const, text }] }; - }), - ); +/** Pair an operation with its formatter, for registerActionTool. */ +export function toolAction<S extends z.ZodObject, O>( + op: Operation<S, O>, + format: Formatter<O>, +): ToolAction { + return { op: op as unknown as Operation, format: format as Formatter<unknown> }; +} - // Tool: search-analytics - server.tool( - "search-analytics", - "View search analytics dashboard and knowledge gap detection", +/** + * Register one tool that runs one of several operations, chosen by its `action` field. + * Each field is optional in the tool schema and validated by the chosen operation. + */ +export function registerActionTool( + server: McpServer, + ctx: OperationContext, + tool: { name: string; description: string; actions: Record<string, ToolAction> }, +): string { + const names = Object.keys(tool.actions) as [string, ...string[]]; + const fields = new Map<string, { schema: z.ZodType; actions: string[] }>(); + for (const [action, { op }] of Object.entries(tool.actions)) { + for (const [key, schema] of Object.entries(shapeOf(op))) { + const field = fields.get(key) ?? { schema, actions: [] }; + field.actions.push(action); + fields.set(key, field); + } + } + const shape: Record<string, z.ZodType> = { + action: z.enum(names).describe(`What to do: ${names.join(", ")}`), + }; + for (const [key, { schema, actions }] of fields) { + shape[key] = schema + .optional() + .describe(`${schema.description ?? key} (action: ${actions.join(", ")})`); + } + server.registerTool( + tool.name, { - days: z.number().optional().describe("Look-back period in days (default: 30)"), - }, - withErrorHandling(async (params) => { - const { getSearchAnalytics, getKnowledgeGaps } = await import("../core/analytics.js"); - const days = params.days ?? 30; - const analytics = getSearchAnalytics(db, days); - const gaps = getKnowledgeGaps(db, days); - - const lines: string[] = [ - `Search Analytics (last ${days} days)`, - `Total searches: ${analytics.totalSearches}`, - `Avg result count: ${analytics.avgResultCount}`, - "", - "Top queries:", - ...analytics.topQueries.map((q) => ` ${q.count}x ${q.query}`), - "", - "Zero-result queries:", - ...analytics.zeroResultQueries.map((q) => ` ${q.count}x ${q.query}`), - "", - "Knowledge gaps:", - ...gaps.map((g) => ` ${g.count}x ${g.query} (last: ${g.lastSearched})`), - ]; - return { content: [{ type: "text" as const, text: lines.join("\n") }] }; + description: tool.description, + inputSchema: z.object(shape), + annotations: toolAnnotations(Object.values(tool.actions).map((a) => a.op.annotations ?? {})), + }, + withErrorHandling(async (args: Record<string, unknown>) => { + const { action, ...input } = args; + const chosen = tool.actions[String(action)]; + if (!chosen) throw new Error(`Unknown action: ${String(action)}`); + return textResult(chosen.format(await runOperation(chosen.op, ctx, input))); }), ); + return tool.name; +} - // Tool: link-documents - server.tool( - "link-documents", - "Create a relationship between two documents (see_also, prerequisite, supersedes, related)", - { - sourceId: z.string().describe("The source document ID"), - targetId: z.string().describe("The target document ID"), - linkType: z - .enum(["see_also", "prerequisite", "supersedes", "related"]) - .describe("Type of relationship"), - label: z.string().optional().describe("Optional human-readable description of the link"), - }, - withErrorHandling((params) => { - const link = createLink( - db, - params.sourceId, - params.targetId, - params.linkType as LinkType, - params.label, - ); - const linkLabel = link.label ? ` — ${link.label}` : ""; - return { - content: [ - { - type: "text" as const, - text: `✓ Link created: ${link.sourceId} → ${link.targetId} (${link.linkType})${linkLabel}`, - }, - ], - }; - }), - ); +/** Optional toolsets enabled by `settings` ("all" enables every one; "core" is always on). */ +export function resolveToolsets(settings: readonly McpToolsetSetting[]): Set<McpToolset> { + const enabled = new Set<McpToolset>(); + for (const name of settings) { + if (name === "all") { + for (const t of MCP_TOOLSETS) enabled.add(t); + } else if (name !== "core") { + enabled.add(name); + } + } + return enabled; +} - // Tool: get-document-links - server.tool( - "get-document-links", - "Get all cross-reference links for a document (both outgoing and incoming)", - { - documentId: z.string().describe("The document ID"), - }, - withErrorHandling((params) => { - const { outgoing, incoming } = getDocumentLinks(db, params.documentId); - if (outgoing.length === 0 && incoming.length === 0) { - return { content: [{ type: "text" as const, text: "No links found for this document." }] }; - } +/** True when `ask` can run: passthrough (the calling assistant answers) or a configured LLM. */ +function askAvailable(ctx: OperationContext): boolean { + return ctx.isPassthrough() || ctx.getLlm() !== null; +} - const lines: string[] = []; - if (outgoing.length > 0) { - lines.push("**Outgoing links:**"); - for (const l of outgoing) { - const outLabel = l.label ? ` — ${l.label}` : ""; - lines.push(` → [${l.linkType}] ${l.targetTitle} (${l.targetId})${outLabel}`); - } - } - if (incoming.length > 0) { - lines.push("**Incoming links:**"); - for (const l of incoming) { - const inLabel = l.label ? ` — ${l.label}` : ""; - lines.push(` ← [${l.linkType}] ${l.sourceTitle} (${l.sourceId})${inLabel}`); - } - } - return { content: [{ type: "text" as const, text: lines.join("\n") }] }; - }), - ); +/** Server `instructions`: how an assistant should use the tools. */ +export function buildInstructions(options: { + ask: boolean; + passthrough: boolean; + admin: boolean; +}): string { + const lines = [ + "libscope is a local knowledge base of documents (library docs, wikis, notes), split into chunks that are searched by meaning and keywords.", + "", + "Workflow:", + "1. search {query} finds relevant chunks. Every result has a documentId and chunkId. Narrow with topic, library, version, sourceType, tags or minRating; page with offset. search {relatedTo: documentId or chunkId} finds similar content.", + "2. get-document {documentId} reads a document with its tags, links and ratings. Long documents are paged: pass maxLength, then the offset shown as 'next page'.", + "3. rate-document {documentId, rating 1-5} after you use a document; add feedback or suggestedCorrection when it is wrong or out of date.", + ]; + if (options.ask) { + lines.push( + options.passthrough + ? "- ask {question} returns the retrieved context without calling an LLM: write the answer yourself and cite the documentIds." + : "- ask {question} answers from the knowledge base with the configured LLM and lists its sources.", + ); + } + lines.push( + "- submit-document adds inline content (with title), a web page (url), a site crawl (url + spider: true) or a public repository URL.", + '- Slow work (crawls, repositories) accepts async: true and returns a taskId. Poll task {"action": "status", taskId} until the status is completed or failed; task {"action": "cancel"} stops it.', + "- overview shows counts, topics, installed packs and index health. list-documents pages through documents (offset, total).", + '- update-document (including tags), delete-document and link-documents {"action": "create" | "delete"} maintain the knowledge base. Deleting cannot be undone.', + ); + if (options.admin) { + lines.push( + "- Admin tools: sync runs saved connector connections (set up with the libscope CLI), install-pack and list-packs manage knowledge packs, reindex-documents re-embeds chunks after an embedding model change.", + ); + } + return lines.join("\n"); +} - // Tool: delete-link - server.tool( - "delete-link", - "Remove a cross-reference link between documents", - { - linkId: z.string().describe("The link ID to delete"), - }, - withErrorHandling((params) => { - deleteLink(db, params.linkId); - return { content: [{ type: "text" as const, text: `✓ Link ${params.linkId} deleted.` }] }; - }), - ); +function packageVersion(): string { + const require = createRequire(import.meta.url); + return (require("../../package.json") as { version: string }).version; +} - // Tool: save-search - server.tool( - "save-search", - "Save a search query with optional filters for later re-use", - { - name: z.string().describe("A unique name for this saved search"), - query: z.string().describe("The search query"), - topic: z.string().optional().describe("Filter by topic ID"), - library: z.string().optional().describe("Filter by library name"), - version: z.string().optional().describe("Filter by library version"), - source: z.string().optional().describe("Filter by source type"), - minRating: z.number().min(1).max(5).optional().describe("Minimum average rating filter"), - limit: z.number().min(1).max(50).optional().describe("Maximum results to return"), - tags: z.array(z.string()).optional().describe("Filter by tags"), - }, - withErrorHandling((params) => { - const { name, query, ...rest } = params; - const filters: Record<string, unknown> = {}; - if (rest.topic !== undefined) filters.topic = rest.topic; - if (rest.library !== undefined) filters.library = rest.library; - if (rest.version !== undefined) filters.version = rest.version; - if (rest.source !== undefined) filters.source = rest.source; - if (rest.minRating !== undefined) filters.minRating = rest.minRating; - if (rest.limit !== undefined) filters.limit = rest.limit; - if (rest.tags !== undefined) filters.tags = rest.tags; - const saved = createSavedSearch( - db, - name, - query, - Object.keys(filters).length > 0 ? filters : undefined, - ); - return { - content: [ - { - type: "text" as const, - text: JSON.stringify(saved, null, 2), - }, - ], - }; - }), - ); +export interface CreateMcpServerOptions extends BootstrapOptions { + /** Use this operation context instead of bootstrapping one. The caller owns its database. */ + ctx?: OperationContext | undefined; + /** Optional toolsets to enable (default: the `mcp.toolsets` config key). */ + toolsets?: readonly McpToolsetSetting[] | undefined; +} - // Tool: suggest-tags - server.tool( - "suggest-tags", - "Suggest tags for a document based on content analysis", - { - documentId: z.string().describe("Document ID"), - maxSuggestions: z.number().min(1).max(20).optional().describe("Max suggestions (default: 5)"), - }, - withErrorHandling((params) => { - const suggestions = suggestTags(db, params.documentId, params.maxSuggestions); - return { - content: [ - { - type: "text" as const, - text: JSON.stringify({ documentId: params.documentId, suggestions }, null, 2), - }, - ], - }; - }), - ); +export interface LibScopeMcpServer { + server: McpServer; + ctx: OperationContext; + /** Names of the registered tools, in registration order. */ + tools: string[]; + /** Close the MCP server, and the database if createMcpServer opened it. */ + close(): Promise<void>; +} - // Tool: list-saved-searches - server.tool( - "list-saved-searches", - "List all saved searches", - {}, - withErrorHandling(() => { - const searches = listSavedSearches(db); - return { - content: [ - { - type: "text" as const, - text: searches.length === 0 ? "No saved searches." : JSON.stringify(searches, null, 2), - }, - ], - }; - }), - ); +type AddTool = <S extends z.ZodObject, O>( + op: Operation<S, O>, + options: OperationToolOptions<O>, +) => void; + +/** Register the 11 core tools (10 when `ask` is unavailable), except `task`. */ +function registerCoreTools(ctx: OperationContext, withAsk: boolean, add: AddTool): void { + add(searchOperation, { + format: fmt.formatSearchResults, + description: + "Search the knowledge base by meaning and keywords (query), or find content similar to a document or chunk (relatedTo). Results carry documentId and chunkId.", + }); + if (withAsk) { + add(askOperation, { + format: fmt.formatAnswer, + description: ctx.isPassthrough() + ? "Retrieve the knowledge-base context for a question (passthrough: no LLM is called; answer from the returned context yourself)" + : askOperation.summary, + }); + } + add(getDocumentOperation, { format: fmt.formatDocumentView }); + add(listDocumentsOperation, { format: fmt.formatDocumentList }); + add(overviewOperation, { format: fmt.formatOverview }); + add(addOperation, { + name: "submit-document", + description: + "Add to the knowledge base: inline content (with title), a web page (url), a site crawl (url + spider: true) or a public GitHub/GitLab repository URL. Local file paths are not accepted.", + omit: ["source", "kind", "format", "include", "exclude", "token"], + format: fmt.formatIngestResult, + }); + add(updateDocumentOperation, { format: fmt.formatDocumentUpdated }); + add(deleteDocumentOperation, { format: fmt.formatDocumentDeleted }); + add(rateDocumentOperation, { format: fmt.formatRating }); +} - // Tool: run-saved-search - server.tool( - "run-saved-search", - "Execute a saved search by name or ID and return results", - { - nameOrId: z.string().describe("The name or ID of the saved search to run"), - }, - withErrorHandling(async (params) => { - const { search, results } = await runSavedSearch(db, provider, params.nameOrId); - return { - content: [ - { - type: "text" as const, - text: JSON.stringify({ search, resultCount: results.length, results }, null, 2), - }, - ], - }; - }), - ); +/** Register the admin toolset. */ +function registerAdminTools(add: AddTool): void { + add(syncOperation, { + description: + "Sync one saved connector connection (name) or all of them (all: true) with the settings saved by 'libscope connect'", + format: fmt.formatSyncResult, + }); + add(installPackOperation, { format: fmt.formatInstallPack }); + add(listPacksOperation, { format: fmt.formatPackList }); + add(reindexOperation, { name: "reindex-documents", format: fmt.formatReindex }); +} - // Tool: delete-saved-search - server.tool( - "delete-saved-search", - "Delete a saved search by name or ID", - { - nameOrId: z.string().describe("The name or ID of the saved search to delete"), - }, - withErrorHandling((params) => { - deleteSavedSearch(db, params.nameOrId); - return { - content: [{ type: "text" as const, text: `✓ Saved search "${params.nameOrId}" deleted.` }], - }; - }), - ); +/** + * Build the libscope MCP server. Without `ctx`, opens the database with `bootstrap(options)`. + * Does not connect a transport. + */ +export function createMcpServer(options: CreateMcpServerOptions = {}): LibScopeMcpServer { + const { ctx: givenCtx, toolsets, ...bootOptions } = options; + let boot: Bootstrapped | undefined; + let ctx = givenCtx; + if (!ctx) { + boot = bootstrap(bootOptions); + ctx = createOperationContext({ + db: boot.db, + provider: boot.provider, + config: boot.config, + surface: "mcp", + }); + } + const opCtx = ctx; + const enabled = resolveToolsets(toolsets ?? opCtx.config.mcp?.toolsets ?? []); + const withAsk = askAvailable(opCtx); + const admin = enabled.has("admin"); - // Tool: create-webhook - server.tool( - "create-webhook", - "Register a webhook to receive notifications for document events", + const server = new McpServer( + { name: "libscope", version: packageVersion() }, { - url: z.string().describe("The URL to send webhook POST requests to (http:// or https://)"), - events: z - .array(z.string()) - .describe( - "Event types to subscribe to: document.created, document.updated, document.deleted, document.rated, search.executed", - ), - secret: z - .string() - .optional() - .describe("Optional secret for HMAC-SHA256 signature verification"), + instructions: buildInstructions({ ask: withAsk, passthrough: opCtx.isPassthrough(), admin }), }, - withErrorHandling(async (params) => { - const webhook = await createWebhook( - db, - params.url, - params.events as WebhookEvent[], - params.secret, - ); - return { - content: [{ type: "text" as const, text: JSON.stringify(redactWebhook(webhook), null, 2) }], - }; - }), ); - // Tool: list-webhooks - server.tool( - "list-webhooks", - "List all registered webhooks", - {}, - withErrorHandling(() => { - const webhooks = listWebhooks(db); - return { - content: [ - { type: "text" as const, text: JSON.stringify(webhooks.map(redactWebhook), null, 2) }, - ], - }; - }), - ); + const tools: string[] = []; + // Formatters of tools that can run in the background, so `task` can format their results. + const taskFormatters = new Map<string, Formatter<unknown>>(); + const add: AddTool = (op, toolOptions) => { + tools.push(registerOperationTool(server, op, opCtx, toolOptions)); + if (op.annotations?.longRunning) { + taskFormatters.set(op.name, toolOptions.format as Formatter<unknown>); + } + }; - // Tool: delete-webhook - server.tool( - "delete-webhook", - "Remove a registered webhook by ID", - { - id: z.string().describe("The webhook ID to delete"), - }, - withErrorHandling((params) => { - deleteWebhook(db, params.id); - return { - content: [{ type: "text" as const, text: `✓ Webhook "${params.id}" deleted.` }], - }; + registerCoreTools(opCtx, withAsk, add); + tools.push( + registerActionTool(server, opCtx, { + name: "link-documents", + description: + "Create a typed link from one document to another (action: create), or delete a link by linkId (action: delete). get-document lists a document's links.", + actions: { + create: toolAction(linkDocumentsOperation, fmt.formatLinkCreated), + delete: toolAction(unlinkDocumentsOperation, fmt.formatLinkDeleted), + }, }), ); - - // Tool: get-task - server.tool( - "get-task", - "Get the current status, progress, and result of an async background task", - { - taskId: z.string().describe("Task ID returned by an async operation"), - }, - withErrorHandling((params) => { - const task = taskRegistry.get(params.taskId); - if (!task) { - return { - content: [ - { - type: "text" as const, - text: `Task ${params.taskId} not found or has expired (tasks are kept for 1 hour after completion).`, - }, - ], - }; - } - return { content: [{ type: "text" as const, text: JSON.stringify(task, null, 2) }] }; + const formatTaskResult: fmt.TaskResultFormatter = (operation, result) => + taskFormatters.get(operation)?.(result); + tools.push( + registerActionTool(server, opCtx, { + name: "task", + description: + "Background tasks started with async: true. status: progress and result of a task; cancel: stop it; list: tasks from the last hour.", + actions: { + status: toolAction(getTaskOperation, (task) => fmt.formatTask(task, formatTaskResult)), + cancel: toolAction(cancelTaskOperation, fmt.formatTaskCancel), + list: toolAction(listTasksOperation, fmt.formatTaskList), + }, }), ); + if (admin) registerAdminTools(add); - // Tool: cancel-task - server.tool( - "cancel-task", - "Request cancellation of a pending or running async background task", - { - taskId: z.string().describe("Task ID to cancel"), + return { + server, + ctx: opCtx, + tools, + close: async (): Promise<void> => { + await server.close(); + boot?.close(); }, - withErrorHandling((params) => { - const outcome = taskRegistry.cancel(params.taskId); - if (outcome === "not_found") { - return { - content: [ - { - type: "text" as const, - text: `Task ${params.taskId} not found or has expired.`, - }, - ], - }; - } - if (outcome === "already_terminal") { - const task = taskRegistry.get(params.taskId); - const status = task?.status ?? "unknown"; - return { - content: [ - { - type: "text" as const, - text: `Task ${params.taskId} cannot be cancelled (current status: ${status}).`, - }, - ], - }; - } - return { - content: [ - { - type: "text" as const, - text: `Cancellation requested for task ${params.taskId}. Running operations will stop at the next checkpoint.`, - }, - ], - }; - }), - ); + }; +} - const transport = new StdioServerTransport(); - await server.connect(transport); +export interface RunStdioServerOptions extends CreateMcpServerOptions { + /** Log level (default: config logging.level). Logs always go to stderr. */ + logLevel?: LogLevel | undefined; } -main().catch((err: unknown) => { - console.error("Fatal error starting LibScope MCP server:", err); - process.exit(1); -}); +/** + * Start the MCP server on stdin/stdout. stdout carries only JSON-RPC: logs go to stderr. + * Closes the server and database on SIGINT/SIGTERM. + */ +export async function runStdioServer( + options: RunStdioServerOptions = {}, +): Promise<LibScopeMcpServer> { + const { logLevel, ...serverOptions } = options; + const config = options.ctx?.config ?? options.config ?? loadConfig(); + initLogger(logLevel ?? config.logging.level, { destination: "stderr" }); + const mcp = createMcpServer({ ...serverOptions, config }); + const shutdown = (): void => { + mcp.close().then( + () => process.exit(0), + () => process.exit(1), + ); + }; + process.once("SIGINT", shutdown); + process.once("SIGTERM", shutdown); + await mcp.server.connect(new StdioServerTransport()); + return mcp; +} diff --git a/src/mcp/tasks.ts b/src/mcp/tasks.ts deleted file mode 100644 index 34795c78..00000000 --- a/src/mcp/tasks.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { randomUUID } from "node:crypto"; - -export type TaskStatus = "pending" | "running" | "completed" | "failed" | "cancelled"; -export type TaskType = "index_document" | "reindex_library" | "sync_connector" | "install_pack"; - -export interface TaskProgress { - current: number; - total: number; - message?: string | undefined; -} - -export interface Task { - id: string; - type: TaskType; - status: TaskStatus; - progress?: TaskProgress | undefined; - result?: string | undefined; - error?: string | undefined; - createdAt: Date; - startedAt?: Date | undefined; - completedAt?: Date | undefined; -} - -/** TTL for completed/failed/cancelled tasks before they are pruned (1 hour). */ -const TASK_TTL_MS = 60 * 60 * 1000; - -export class TaskRegistry { - private readonly tasks = new Map<string, Task>(); - private readonly controllers = new Map<string, AbortController>(); - - /** Create a new task and return it along with its AbortSignal. */ - create(type: TaskType): { task: Task; signal: AbortSignal } { - const id = randomUUID(); - const task: Task = { - id, - type, - status: "pending", - createdAt: new Date(), - }; - const controller = new AbortController(); - this.tasks.set(id, task); - this.controllers.set(id, controller); - return { task, signal: controller.signal }; - } - - /** Retrieve a task by ID. Returns undefined if not found or expired. */ - get(id: string): Task | undefined { - this.prune(); - return this.tasks.get(id); - } - - /** Apply partial updates to a task. No-op if task not found. */ - update(id: string, updates: Partial<Task>): void { - const task = this.tasks.get(id); - if (task) { - Object.assign(task, updates); - } - } - - /** - * Attempt to cancel a task. - * Returns: - * "cancelled" — cancellation was requested - * "not_found" — task ID unknown or expired - * "already_terminal" — task already completed, failed, or cancelled - */ - cancel(id: string): "cancelled" | "not_found" | "already_terminal" { - this.prune(); - const task = this.tasks.get(id); - if (!task) return "not_found"; - if (task.status === "completed" || task.status === "failed" || task.status === "cancelled") { - return "already_terminal"; - } - this.controllers.get(id)?.abort(); - if (task.status === "pending") { - task.status = "cancelled"; - task.completedAt = new Date(); - } - // Running tasks detect abort via signal and update their own status. - return "cancelled"; - } - - /** Remove expired completed/failed/cancelled tasks. */ - private prune(): void { - const cutoff = Date.now() - TASK_TTL_MS; - for (const [id, task] of this.tasks) { - if (task.completedAt && task.completedAt.getTime() < cutoff) { - this.tasks.delete(id); - this.controllers.delete(id); - } - } - } -} - -/** Module-level singleton task registry used by the MCP server. */ -export const taskRegistry = new TaskRegistry(); diff --git a/src/providers/dimensions.ts b/src/providers/dimensions.ts new file mode 100644 index 00000000..16fdbe55 --- /dev/null +++ b/src/providers/dimensions.ts @@ -0,0 +1,77 @@ +import { ConfigError, EmbeddingError } from "../errors.js"; + +/** Output dimensions of common embedding models, keyed by model name (without a tag). */ +const KNOWN_MODEL_DIMENSIONS: ReadonlyMap<string, number> = new Map([ + // OpenAI + ["text-embedding-3-small", 1536], + ["text-embedding-3-large", 3072], + ["text-embedding-ada-002", 1536], + // Ollama + ["nomic-embed-text", 768], + ["mxbai-embed-large", 1024], + ["all-minilm", 384], + ["bge-m3", 1024], + ["bge-large", 1024], + ["snowflake-arctic-embed2", 1024], + ["paraphrase-multilingual", 768], +]); + +/** Largest dimension count accepted for the vector table. */ +export const MAX_EMBEDDING_DIMENSIONS = 10000; + +/** + * Look up the output dimension of a known model. Ollama tags (`name:tag`) are ignored, + * because every tag of the models in the table has the same dimension. + */ +export function knownModelDimensions(model: string): number | undefined { + return KNOWN_MODEL_DIMENSIONS.get(model) ?? KNOWN_MODEL_DIMENSIONS.get(model.split(":")[0] ?? ""); +} + +/** Throw a ConfigError unless `value` is a usable `embedding.dimensions` value. */ +export function validateDimensionsOverride(value: unknown): number | undefined { + if (value === undefined) return undefined; + if ( + typeof value !== "number" || + !Number.isInteger(value) || + value <= 0 || + value > MAX_EMBEDDING_DIMENSIONS + ) { + throw new ConfigError( + `embedding.dimensions must be a positive integer <= ${MAX_EMBEDDING_DIMENSIONS}, got ${JSON.stringify(value)}`, + ); + } + return value; +} + +/** + * Tracks the expected vector size for a provider. The expected size comes from the + * `embedding.dimensions` override, else the known-model table, else the first response. + */ +export class EmbeddingDimensions { + private expected: number | undefined; + + constructor( + private readonly model: string, + override?: number, + ) { + this.expected = override ?? knownModelDimensions(model); + } + + /** Expected dimension, or 0 while it is unknown (before the first embedding). */ + get value(): number { + return this.expected ?? 0; + } + + /** Check every vector against the expected size; learn the size from the first vector. */ + check(vectors: number[][]): void { + for (const vector of vectors) { + this.expected ??= vector.length; + if (vector.length !== this.expected) { + throw new EmbeddingError( + `Expected embedding dimension ${this.expected}, got ${vector.length} from model "${this.model}". ` + + `If this model produces ${vector.length}-dimensional vectors, set embedding.dimensions to ${vector.length}.`, + ); + } + } + } +} diff --git a/src/providers/embedding.ts b/src/providers/embedding.ts index 7c5652df..8bc30d02 100644 --- a/src/providers/embedding.ts +++ b/src/providers/embedding.ts @@ -6,7 +6,13 @@ export interface EmbeddingProvider { /** Provider name for display/logging. */ readonly name: string; - /** Dimensionality of the output vectors. */ + /** Model identifier, recorded with the vector index to detect model changes. */ + readonly model?: string | undefined; + + /** + * Dimensionality of the output vectors. + * 0 means not yet known: the provider learns it from the first embedding it returns. + */ readonly dimensions: number; /** Generate an embedding vector for a single text input. */ diff --git a/src/providers/index.ts b/src/providers/index.ts index 4676b7e3..b041fc74 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -3,54 +3,39 @@ export { LocalEmbeddingProvider } from "./local.js"; export { OllamaEmbeddingProvider } from "./ollama.js"; export { OpenAIEmbeddingProvider } from "./openai.js"; -import type { LibScopeConfig } from "../config.js"; +import { DEFAULT_OLLAMA_URL, type LibScopeConfig } from "../config.js"; import { ConfigError } from "../errors.js"; +import { validateDimensionsOverride } from "./dimensions.js"; import type { EmbeddingProvider } from "./embedding.js"; import { LocalEmbeddingProvider } from "./local.js"; import { OllamaEmbeddingProvider } from "./ollama.js"; import { OpenAIEmbeddingProvider } from "./openai.js"; -/** Factory function that creates an EmbeddingProvider from config. */ -export type ProviderFactory = (config: LibScopeConfig) => EmbeddingProvider; - -const providerRegistry = new Map<string, ProviderFactory>(); - -/** Register a custom embedding provider factory. */ -export function registerProvider(name: string, factory: ProviderFactory): void { - providerRegistry.set(name, factory); -} - /** Create an embedding provider based on config. */ export function createEmbeddingProvider(config: LibScopeConfig): EmbeddingProvider { - const registered = providerRegistry.get(config.embedding.provider); - if (registered) { - return registered(config); - } - - switch (config.embedding.provider) { + const { embedding } = config; + switch (embedding.provider) { case "local": return new LocalEmbeddingProvider(); - case "ollama": { - if (!config.embedding.ollamaUrl) { - throw new ConfigError( - "Ollama URL is required. Set LIBSCOPE_OLLAMA_URL or configure in ~/.libscope/config.json", - ); - } - if (!config.embedding.ollamaModel) { - throw new ConfigError( - "Ollama model is required. Set LIBSCOPE_OLLAMA_MODEL or configure in ~/.libscope/config.json", - ); - } - return new OllamaEmbeddingProvider(config.embedding.ollamaUrl, config.embedding.ollamaModel); - } + case "ollama": + return new OllamaEmbeddingProvider( + embedding.url ?? DEFAULT_OLLAMA_URL, + embedding.model ?? "nomic-embed-text", + validateDimensionsOverride(embedding.dimensions), + ); case "openai": { - const apiKey = config.embedding.openaiApiKey; + const apiKey = config.openai?.apiKey; if (!apiKey) { throw new ConfigError( - "OpenAI API key is required. Set LIBSCOPE_OPENAI_API_KEY or configure in ~/.libscope/config.json", + "OpenAI API key is required. Set LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY, " + + 'or run "libscope config set openai.apiKey <key>".', ); } - return new OpenAIEmbeddingProvider(apiKey, config.embedding.openaiModel); + return new OpenAIEmbeddingProvider( + apiKey, + embedding.model, + validateDimensionsOverride(embedding.dimensions), + ); } default: throw new ConfigError(`Unknown embedding provider: ${String(config.embedding.provider)}`); diff --git a/src/providers/local.ts b/src/providers/local.ts index 37391b32..24873138 100644 --- a/src/providers/local.ts +++ b/src/providers/local.ts @@ -1,6 +1,9 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; import { EmbeddingError } from "../errors.js"; import type { EmbeddingProvider } from "./embedding.js"; import { getLogger } from "../logger.js"; +import { mapSequential } from "../utils/async.js"; /** Minimal typed interface for the @xenova/transformers feature-extraction pipeline output. */ interface TransformersOutput { @@ -13,17 +16,30 @@ type FeatureExtractionPipeline = ( options: { pooling: string; normalize: boolean }, ) => Promise<TransformersOutput>; +/** Download events from @xenova/transformers while the model is fetched (first use only). */ +export interface ModelDownloadEvent { + /** "initiate" | "download" | "progress" | "done" per file, then "ready". */ + status: string; + file?: string | undefined; + /** Percent of `file` downloaded (status "progress"). */ + progress?: number | undefined; +} + /** * Local embedding provider using @xenova/transformers (all-MiniLM-L6-v2). * Downloads the model on first use (~80MB). Runs entirely in-process. */ export class LocalEmbeddingProvider implements EmbeddingProvider { readonly name = "local"; + readonly model = "Xenova/all-MiniLM-L6-v2"; readonly dimensions = 384; private pipeline: FeatureExtractionPipeline | null = null; private initPromise: Promise<void> | null = null; + /** Called while the model is downloaded (not when it is already cached), e.g. to show progress. */ + onDownloadProgress: ((event: ModelDownloadEvent) => void) | undefined; + private async ensureInitialized(): Promise<void> { this.initPromise ??= this.doInitialize(); await this.initPromise; @@ -34,11 +50,15 @@ export class LocalEmbeddingProvider implements EmbeddingProvider { log.info("Loading local embedding model (all-MiniLM-L6-v2)..."); try { // Dynamic import to avoid loading transformers until needed - const { pipeline } = await import("@xenova/transformers"); + const { pipeline, env } = await import("@xenova/transformers"); + const cacheDir: unknown = env.cacheDir; + const cached = typeof cacheDir === "string" && existsSync(join(cacheDir, this.model)); + const onDownload = cached ? undefined : this.onDownloadProgress; // Cast to the typed interface; @xenova/transformers lacks precise TS generics for pipeline output this.pipeline = (await pipeline( "feature-extraction", - "Xenova/all-MiniLM-L6-v2", + this.model, + onDownload ? { progress_callback: onDownload } : undefined, )) as unknown as FeatureExtractionPipeline; log.info("Local embedding model loaded successfully"); } catch (err) { @@ -76,10 +96,6 @@ export class LocalEmbeddingProvider implements EmbeddingProvider { } } // Process sequentially to avoid memory issues with local model - const results: number[][] = []; - for (const text of texts) { - results.push(await this.embed(text)); - } - return results; + return await mapSequential(texts, (text) => this.embed(text)); } } diff --git a/src/providers/ollama.ts b/src/providers/ollama.ts index 79d2b32d..8bc48b1b 100644 --- a/src/providers/ollama.ts +++ b/src/providers/ollama.ts @@ -1,6 +1,7 @@ import { EmbeddingError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { withRetry } from "../utils/retry.js"; +import { EmbeddingDimensions } from "./dimensions.js"; import type { EmbeddingProvider } from "./embedding.js"; /** @@ -9,14 +10,23 @@ import type { EmbeddingProvider } from "./embedding.js"; */ export class OllamaEmbeddingProvider implements EmbeddingProvider { readonly name = "ollama"; - readonly dimensions: number; + private readonly expectedDimensions: EmbeddingDimensions; + /** + * @param dimensions - Override for the vector size. When omitted, known models use their + * documented size and other models use the size of the first embedding returned. + */ constructor( private readonly baseUrl: string = "http://localhost:11434", - private readonly model: string = "nomic-embed-text", - dimensions: number = 768, + readonly model: string = "nomic-embed-text", + dimensions?: number, ) { - this.dimensions = dimensions; + this.expectedDimensions = new EmbeddingDimensions(model, dimensions); + } + + /** Vector size, or 0 until the first embedding of an unknown model. */ + get dimensions(): number { + return this.expectedDimensions.value; } async embed(text: string): Promise<number[]> { @@ -62,11 +72,7 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider { if (!embedding) { throw new EmbeddingError("Ollama returned empty embeddings"); } - if (embedding.length !== this.dimensions) { - throw new EmbeddingError( - `Expected embedding dimension ${this.dimensions}, got ${embedding.length}`, - ); - } + this.expectedDimensions.check([embedding]); return embedding; }); } catch (err) { @@ -129,13 +135,7 @@ export class OllamaEmbeddingProvider implements EmbeddingProvider { `Ollama returned ${embeddings.length} embeddings for ${texts.length} inputs`, ); } - for (const emb of embeddings) { - if (emb.length !== this.dimensions) { - throw new EmbeddingError( - `Expected embedding dimension ${this.dimensions}, got ${emb.length}`, - ); - } - } + this.expectedDimensions.check(embeddings); return embeddings; }); } catch (err) { diff --git a/src/providers/openai.ts b/src/providers/openai.ts index b74ad93d..24232df4 100644 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -2,6 +2,7 @@ import OpenAI from "openai"; import { EmbeddingError } from "../errors.js"; import { createChildLogger } from "../logger.js"; import { withRetry } from "../utils/retry.js"; +import { EmbeddingDimensions } from "./dimensions.js"; import type { EmbeddingProvider } from "./embedding.js"; /** @@ -10,15 +11,30 @@ import type { EmbeddingProvider } from "./embedding.js"; */ export class OpenAIEmbeddingProvider implements EmbeddingProvider { readonly name = "openai"; - readonly dimensions = 1536; private readonly client: OpenAI; + private readonly expectedDimensions: EmbeddingDimensions; + /** Sent as the API `dimensions` parameter (text-embedding-3 models can shorten vectors). */ + private readonly requestDimensions: { dimensions?: number }; + /** + * @param dimensions - Override for the vector size. For text-embedding-3 models it is also + * sent to the API to request shortened vectors. + */ constructor( apiKey: string, - private readonly model: string = "text-embedding-3-small", + readonly model: string = "text-embedding-3-small", + dimensions?: number, ) { this.client = new OpenAI({ apiKey, timeout: 30_000 }); + this.expectedDimensions = new EmbeddingDimensions(model, dimensions); + this.requestDimensions = + dimensions !== undefined && model.startsWith("text-embedding-3") ? { dimensions } : {}; + } + + /** Vector size, or 0 until the first embedding of an unknown model. */ + get dimensions(): number { + return this.expectedDimensions.value; } async embed(text: string): Promise<number[]> { @@ -31,16 +47,13 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { const response = await this.client.embeddings.create({ model: this.model, input: text, + ...this.requestDimensions, }); const embedding = response.data[0]?.embedding; if (!embedding) { throw new EmbeddingError("OpenAI returned empty embedding"); } - if (embedding.length !== this.dimensions) { - throw new EmbeddingError( - `Expected embedding dimension ${this.dimensions}, got ${embedding.length}`, - ); - } + this.expectedDimensions.check([embedding]); return embedding; }); } catch (err) { @@ -67,6 +80,7 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { const response = await this.client.embeddings.create({ model: this.model, input: texts, + ...this.requestDimensions, }); if (response.data.length !== texts.length) { throw new EmbeddingError( @@ -74,13 +88,7 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider { ); } const embeddings = response.data.map((d) => d.embedding); - for (const emb of embeddings) { - if (emb.length !== this.dimensions) { - throw new EmbeddingError( - `Expected embedding dimension ${this.dimensions}, got ${emb.length}`, - ); - } - } + this.expectedDimensions.check(embeddings); return embeddings; }); } catch (err) { diff --git a/src/registry/config.ts b/src/registry/config.ts index 21dcad83..3fea3b74 100644 --- a/src/registry/config.ts +++ b/src/registry/config.ts @@ -3,18 +3,12 @@ * Reads/writes the "registries" array in ~/.libscope/config.json. */ -import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; -import { ConfigError, ValidationError } from "../errors.js"; +import { readRawUserConfig, writeRawUserConfig } from "../config.js"; +import { NotFoundError, ValidationError } from "../errors.js"; import { getLogger } from "../logger.js"; +import { trimTrailingSlashes } from "../utils/strings.js"; import type { RegistryEntry } from "./types.js"; -/** Path to the user config file. */ -function getUserConfigPath(): string { - return join(homedir(), ".libscope", "config.json"); -} - /** Sanitize a URL for safe display in logs — masks any embedded credentials. */ export function sanitizeUrl(url: string): string { // Replace password in https://user:pass@host or https://token@host patterns @@ -40,58 +34,75 @@ export function validateRegistryName(name: string): void { } } -/** Validate a git URL (https, ssh://, or SCP-style). Returns the normalized (trimmed, no trailing slash) URL. */ +/** True when `word` ends right before index `end` of `text`. */ +function endsAt(text: string, word: string, end: number): boolean { + return end >= word.length && text.startsWith(word, end - word.length); +} + +/** + * True when an "http://" or "https://" anywhere in `url` is followed by user info: one or more + * characters other than "@" and "/", then "@" (https://user:pass@host, https://token@host). + * A linear scan, so a long hostile URL cannot make the check slow. + */ +function hasEmbeddedCredentials(url: string): boolean { + for (let sep = url.indexOf("://"); sep !== -1; sep = url.indexOf("://", sep + 1)) { + if (endsAt(url, "http", sep) || endsAt(url, "https", sep)) { + const start = sep + 3; + let end = start; + while (end < url.length && url.charAt(end) !== "@" && url.charAt(end) !== "/") end++; + if (end > start && url.charAt(end) === "@") return true; + } + } + return false; +} + +/** + * Validate a git URL: https://, ssh://, SCP-style git@host:path, or file:/// for a repository on + * a local or shared disk. Returns the normalized (trimmed, no trailing slash) URL. + */ export function validateGitUrl(url: string): string { // Trim whitespace and trailing slashes - const normalized = url.trim().replace(/\/+$/, ""); + const normalized = trimTrailingSlashes(url.trim()); // Reject URLs with embedded credentials (e.g. https://user:pass@host or https://token@host) - if (/https?:\/\/[^@/]+:[^@/]*@/.test(normalized) || /https?:\/\/[^@/]+@/.test(normalized)) { + if (hasEmbeddedCredentials(normalized)) { throw new ValidationError( "Registry URL must not contain embedded credentials (user:pass@host or token@host). " + "Use SSH keys or a git credential helper instead.", ); } - // Accept https:// URLs, ssh:// URLs, and SCP-style git@host:path URLs const isHttps = normalized.startsWith("https://"); const isSshProtocol = normalized.startsWith("ssh://"); const isScp = /^git@[\w.-]+:/.test(normalized); - if (!isHttps && !isSshProtocol && !isScp) { + const isFile = normalized.startsWith("file:///"); + if (!isHttps && !isSshProtocol && !isScp && !isFile) { throw new ValidationError( - "Registry URL must use https://, ssh://, or SSH (git@host:path) format", + "Registry URL must use https://, ssh://, SSH (git@host:path) or file:/// format", ); } return normalized; } -/** Read the raw config JSON from disk. */ -function readRawConfig(): Record<string, unknown> { - const configPath = getUserConfigPath(); - if (!existsSync(configPath)) return {}; +/** Derive a registry name from a git URL ("https://github.com/org/packs.git" -> "packs"). */ +export function deriveRegistryName(url: string): string { + const trimmed = trimTrailingSlashes(url.trim()); + const last = trimmed.slice(Math.max(trimmed.lastIndexOf("/"), trimmed.lastIndexOf(":")) + 1); + const name = last.endsWith(".git") ? last.slice(0, -4) : last; try { - const raw = readFileSync(configPath, "utf-8"); - return JSON.parse(raw) as Record<string, unknown>; - } catch (err) { - throw new ConfigError("Failed to read config file", err); - } -} - -/** Write the raw config JSON to disk, preserving all other keys. */ -function writeRawConfig(config: Record<string, unknown>): void { - const dir = join(homedir(), ".libscope"); - if (!existsSync(dir)) { - mkdirSync(dir, { recursive: true, mode: 0o700 }); + validateRegistryName(name); + } catch { + throw new ValidationError( + `Could not derive a registry name from "${url}" (got "${name}"). Give a name.`, + ); } - const configPath = getUserConfigPath(); - writeFileSync(configPath, JSON.stringify(config, null, 2), "utf-8"); - chmodSync(configPath, 0o600); + return name; } /** Load all registry entries from config. */ export function loadRegistries(): RegistryEntry[] { - const config = readRawConfig(); + const config = readRawUserConfig(); const registries = config["registries"]; if (!Array.isArray(registries)) return []; return registries as RegistryEntry[]; @@ -99,9 +110,9 @@ export function loadRegistries(): RegistryEntry[] { /** Save registry entries to config (merges with existing config keys). */ export function saveRegistries(registries: RegistryEntry[]): void { - const config = readRawConfig(); + const config = readRawUserConfig(); config["registries"] = registries; - writeRawConfig(config); + writeRawUserConfig(config); } /** Find a registry by name. Returns undefined if not found. */ @@ -109,13 +120,23 @@ export function getRegistry(name: string): RegistryEntry | undefined { return loadRegistries().find((r) => r.name === name); } -/** Add a new registry entry. Throws if name already exists. */ -export function addRegistry(entry: RegistryEntry): void { +/** The registry called `name`; NotFoundError when none is configured. */ +export function requireRegistry(name: string): RegistryEntry { + const entry = getRegistry(name); + if (!entry) { + throw new NotFoundError( + `Registry "${name}" not found. Add it with: libscope registry add <url> --name ${name}`, + ); + } + return entry; +} + +/** Add a new registry entry. Throws if the name already exists. Returns the stored entry. */ +export function addRegistry(entry: RegistryEntry): RegistryEntry { const log = getLogger(); validateRegistryName(entry.name); // Normalize and validate URL; use the normalized form going forward - const normalizedUrl = validateGitUrl(entry.url); - const normalizedEntry = { ...entry, url: normalizedUrl }; + const normalizedEntry = { ...entry, url: validateGitUrl(entry.url) }; const registries = loadRegistries(); if (registries.some((r) => r.name === normalizedEntry.name)) { @@ -128,19 +149,14 @@ export function addRegistry(entry: RegistryEntry): void { { registry: normalizedEntry.name, url: sanitizeUrl(normalizedEntry.url) }, "Registry added to config", ); + return normalizedEntry; } -/** Remove a registry entry by name. Throws if not found. */ +/** Remove a registry entry by name. NotFoundError when it is not configured. */ export function removeRegistry(name: string): void { - const log = getLogger(); - const registries = loadRegistries(); - const index = registries.findIndex((r) => r.name === name); - if (index === -1) { - throw new ValidationError(`Registry "${name}" not found`); - } - registries.splice(index, 1); - saveRegistries(registries); - log.info({ registry: name }, "Registry removed from config"); + requireRegistry(name); + saveRegistries(loadRegistries().filter((r) => r.name !== name)); + getLogger().info({ registry: name }, "Registry removed from config"); } /** Update the lastSyncedAt timestamp for a registry. */ diff --git a/src/registry/git.ts b/src/registry/git.ts index fe8443c5..8c32c463 100644 --- a/src/registry/git.ts +++ b/src/registry/git.ts @@ -8,7 +8,7 @@ import { promisify } from "node:util"; import { existsSync, readFileSync, writeFileSync, mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { getLogger } from "../logger.js"; -import { FetchError, ValidationError } from "../errors.js"; +import { ConfigError, FetchError, ValidationError } from "../errors.js"; import type { PackSummary } from "./types.js"; import { INDEX_FILE, PACKS_DIR } from "./types.js"; @@ -35,6 +35,10 @@ export async function git( const { stdout } = await execFile("git", args, { cwd, timeout }); return stdout.trim(); } catch (err) { + // ENOENT with an existing (or no) working directory: the git binary itself is missing. + if ((err as NodeJS.ErrnoException).code === "ENOENT" && (!cwd || existsSync(cwd))) { + throw new ConfigError("git is not installed or not in PATH", err); + } const message = err instanceof Error ? err.message : String(err); log.error({ args, cwd, err: message }, "Git command failed"); @@ -192,8 +196,9 @@ export async function createRegistryRepo(path: string): Promise<void> { mkdirSync(path, { recursive: true }); - // Initialize git repo + // Initialize git repo on "main" whatever init.defaultBranch is (the docs push "main") await git(["init"], { cwd: path }); + await git(["symbolic-ref", "HEAD", "refs/heads/main"], { cwd: path }); // Create canonical structure const indexPath = join(path, INDEX_FILE); @@ -224,22 +229,17 @@ export async function createRegistryRepo(path: string): Promise<void> { log.info({ path }, "Registry repo initialized"); } -/** Check if git is available on the system. */ -export async function checkGitAvailable(): Promise<boolean> { - try { - await execFile("git", ["--version"]); - return true; - } catch { - return false; - } -} - -/** Add, commit, and push changes in a registry repo. */ -export async function commitAndPush(repoPath: string, message: string): Promise<void> { +/** Add and commit all changes in a registry repo. */ +export async function commitAll(repoPath: string, message: string): Promise<void> { await git(["add", "."], { cwd: repoPath }); await git( ["-c", "user.name=libscope", "-c", "user.email=libscope@localhost", "commit", "-m", message], { cwd: repoPath }, ); - await git(["push"], { cwd: repoPath }); +} + +/** Add, commit, and push the current branch of a registry repo (also when the branch is new). */ +export async function commitAndPush(repoPath: string, message: string): Promise<void> { + await commitAll(repoPath, message); + await git(["push", "-u", "origin", "HEAD"], { cwd: repoPath }); } diff --git a/src/registry/publish.ts b/src/registry/publish.ts index cb0ba342..80cf85dc 100644 --- a/src/registry/publish.ts +++ b/src/registry/publish.ts @@ -4,9 +4,8 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync, copyFileSync, rmSync } from "node:fs"; import { join } from "node:path"; -import { gunzipSync } from "node:zlib"; import { getLogger } from "../logger.js"; -import { ValidationError } from "../errors.js"; +import { NotFoundError, ValidationError } from "../errors.js"; import type { PublishOptions, PublishResult, @@ -22,10 +21,10 @@ import { INDEX_FILE, getRegistryCacheDir, } from "./types.js"; -import { getRegistry } from "./config.js"; -import { commitAndPush, fetchRegistry, git, clearIndexCache } from "./git.js"; +import { requireRegistry } from "./config.js"; +import { commitAll, commitAndPush, fetchRegistry, git, clearIndexCache } from "./git.js"; import { computeChecksum, writeChecksumFile } from "./checksum.js"; -import type { KnowledgePack } from "../core/packs.js"; +import { readPackFile, type KnowledgePack } from "../core/packs.js"; /** Remove an entire pack directory and its entry from the index. */ function removeEntirePack(packDir: string, cacheDir: string, packName: string): void { @@ -100,28 +99,17 @@ function bumpPatchVersion(version: string): string { return `${parts[0]}.${parts[1]}.${Number.isNaN(patch) ? 1 : patch + 1}`; } -/** Gzip magic number: first two bytes of a gzip stream. */ -const GZIP_MAGIC = Buffer.from([0x1f, 0x8b]); - /** * Read a pack JSON file (plain or gzip-compressed). * Auto-detects gzip by checking for magic bytes. */ function readPackJson(filePath: string): KnowledgePack { - const raw = readFileSync(filePath); - const text = - raw.length >= 2 && raw[0] === GZIP_MAGIC[0] && raw[1] === GZIP_MAGIC[1] - ? gunzipSync(raw).toString("utf-8") - : raw.toString("utf-8"); - return JSON.parse(text) as KnowledgePack; + return JSON.parse(readPackFile(filePath)) as KnowledgePack; } /** Validate that the registry exists and has a local cache; return the cache directory. */ function validateRegistryCache(registryName: string): string { - const entry = getRegistry(registryName); - if (!entry) { - throw new ValidationError(`Registry "${registryName}" not found. Add it first.`); - } + requireRegistry(registryName); const cacheDir = getRegistryCacheDir(registryName); if (!existsSync(cacheDir)) { throw new ValidationError( @@ -192,7 +180,7 @@ function resolveVersionAndManifest( if (manifest.versions.some((v) => v.version === version)) { throw new ValidationError( `Version ${version} of "${pack.name}" already exists in "${registryName}". ` + - "Use --version to specify a different version.", + "Publish it with a different version.", ); } @@ -256,18 +244,35 @@ function rollbackVersionDir(versionDir: string, created: boolean): void { } } +/** Where a publish goes: its default commit message and how the commit is pushed. */ +interface PublishTarget { + defaultMessage: (packName: string, version: string) => string; + push: (cacheDir: string) => Promise<unknown>; +} + /** * Publish a pack to a registry. * Creates the canonical folder structure, generates checksum, updates index and manifest, commits and pushes. */ export async function publishPack(options: PublishOptions): Promise<PublishResult> { - const log = getLogger(); - const { registryName, packFilePath, commitMessage } = options; - - const cacheDir = validateRegistryCache(registryName); + const cacheDir = validateRegistryCache(options.registryName); await tryFetchLatest(cacheDir, "publish"); + const pack = readAndValidatePack(options.packFilePath); + return publishInCache(options, cacheDir, pack, { + defaultMessage: (name, version) => `publish: ${name}@${version}`, + push: (dir) => git(["push"], { cwd: dir }), + }); +} - const pack = readAndValidatePack(packFilePath); +/** Write the pack's version into the registry clone at `cacheDir`, commit it and push it. */ +async function publishInCache( + options: PublishOptions, + cacheDir: string, + pack: KnowledgePack, + target: PublishTarget, +): Promise<PublishResult> { + const log = getLogger(); + const { registryName, packFilePath, commitMessage } = options; const packDir = join(cacheDir, PACKS_DIR, pack.name); const manifestPath = join(packDir, PACK_MANIFEST_FILE); @@ -321,8 +326,8 @@ export async function publishPack(options: PublishOptions): Promise<PublishResul }; upsertIndexEntry(indexPath, index, summary); - const msg = commitMessage ?? `publish: ${pack.name}@${version}`; - await commitAndPush(cacheDir, msg); + await commitAll(cacheDir, commitMessage ?? target.defaultMessage(pack.name, version)); + await target.push(cacheDir); log.info({ registry: registryName, pack: pack.name, version, checksum }, "Pack published"); return { packName: pack.name, version, checksum, registryName }; @@ -333,47 +338,57 @@ export async function publishPack(options: PublishOptions): Promise<PublishResul } /** - * Publish to a feature branch instead of main (for PR workflow). + * Publish to a `feature/add-<pack>` branch of the registry instead of its main branch (for a + * pull request). The branch starts at the remote's latest main branch and is pushed to the + * remote. The local clone is left on its own branch, so later publishes and syncs are not + * affected. */ export async function publishPackToBranch( options: PublishOptions, ): Promise<PublishResult & { branch: string }> { const log = getLogger(); - const { registryName, packFilePath } = options; + const { registryName } = options; const cacheDir = validateRegistryCache(registryName); - - const pack = readPackJson(packFilePath); + await tryFetchLatest(cacheDir, "publish"); + const pack = readAndValidatePack(options.packFilePath); const branchName = `feature/add-${pack.name}`; - // Create and checkout branch - await git(["checkout", "-b", branchName], { cwd: cacheDir }); + const startRef = await currentRef(cacheDir); + await git(["checkout", "-B", branchName], { cwd: cacheDir }); try { - // Reuse the normal publish flow (which commits) - const result = await publishPack({ - ...options, - commitMessage: options.commitMessage ?? `feat: add ${pack.name}@${pack.version}`, + const result = await publishInCache(options, cacheDir, pack, { + defaultMessage: (name, version) => `feat: add ${name}@${version}`, + push: (dir) => git(["push", "origin", `HEAD:refs/heads/${branchName}`], { cwd: dir }), }); - - // Push the branch - await git(["push", "-u", "origin", branchName], { cwd: cacheDir }); - log.info({ branch: branchName, registry: registryName }, "Pack published to feature branch"); - return { ...result, branch: branchName }; + } finally { + await leaveFeatureBranch(cacheDir, startRef, branchName); + } +} + +/** The checked-out branch name, or the commit when HEAD is detached. */ +async function currentRef(cacheDir: string): Promise<string> { + const branch = await git(["rev-parse", "--abbrev-ref", "HEAD"], { cwd: cacheDir }); + return branch === "HEAD" ? git(["rev-parse", "HEAD"], { cwd: cacheDir }) : branch; +} + +/** Check out `startRef` again, dropping the publish's files, and delete the local branch. */ +async function leaveFeatureBranch( + cacheDir: string, + startRef: string, + featureBranch: string, +): Promise<void> { + try { + await git(["checkout", "-f", startRef], { cwd: cacheDir }); + await git(["branch", "-D", featureBranch], { cwd: cacheDir }); } catch (err) { - // Try to go back to main branch on failure - try { - await git(["checkout", "main"], { cwd: cacheDir }); - await git(["branch", "-D", branchName], { cwd: cacheDir }); - } catch (cleanupErr) { - log.warn( - { err: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr) }, - "Failed to clean up feature branch after publish failure", - ); - } - throw err; + getLogger().warn( + { err: err instanceof Error ? err.message : String(err) }, + "Failed to return the registry clone to its branch after publishing to a feature branch", + ); } } @@ -395,13 +410,13 @@ export async function unpublishPack(options: UnpublishOptions): Promise<void> { const manifestPath = join(packDir, PACK_MANIFEST_FILE); if (!existsSync(manifestPath)) { - throw new ValidationError(`Pack "${packName}" not found in registry "${registryName}".`); + throw new NotFoundError(`Pack "${packName}" not found in registry "${registryName}".`); } const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as PackManifest; const versionIdx = manifest.versions.findIndex((v) => v.version === version); if (versionIdx === -1) { - throw new ValidationError( + throw new NotFoundError( `Version ${version} of "${packName}" not found in registry "${registryName}".`, ); } diff --git a/src/registry/resolve.ts b/src/registry/resolve.ts index 6eac6efb..fa7e96b6 100644 --- a/src/registry/resolve.ts +++ b/src/registry/resolve.ts @@ -1,28 +1,17 @@ /** - * Registry pack resolution: find and resolve a pack from configured registries. + * Registry pack resolution: turn "name" or "name@version" into a pack file in the local copy + * of one configured registry. No network access; run `libscope registry sync` to update. */ import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; import { getLogger } from "../logger.js"; -import { ValidationError } from "../errors.js"; -import type { - RegistryEntry, - PackSummary, - PackManifest, - RegistryConflict, - ConflictResolution, -} from "./types.js"; -import { - getRegistryCacheDir, - getPackManifestPath, - getPackDataPath, - PACK_MANIFEST_FILE, -} from "./types.js"; +import { NotFoundError, ValidationError } from "../errors.js"; +import type { PackManifest } from "./types.js"; +import { getPackManifestPath, getPackDataPath } from "./types.js"; import { loadRegistries } from "./config.js"; -import { readIndex } from "./git.js"; import { verifyChecksum } from "./checksum.js"; import { validatePathSegment } from "./publish.js"; +import { listRegistryPacks } from "./search.js"; /** Parse a pack specifier like "name@1.2.0" into name and optional version. */ export function parsePackSpecifier(specifier: string): { name: string; version?: string } { @@ -36,77 +25,77 @@ export function parsePackSpecifier(specifier: string): { name: string; version?: return { name: specifier }; } -/** Result of resolving a pack from registries. */ +/** A pack found in a registry. */ export interface ResolvedPack { - registryName: string; - registryUrl: string; - packName: string; + registry: string; + name: string; version: string; - /** Path to the pack data file in the local cache. */ + /** Path to the pack data file in the local copy of the registry. */ dataPath: string; } /** - * Find all registries that have a pack with the given name. + * Find "name" or "name@version" in the configured registries (only `registryName` when given). + * Without a version, the registry's latest version is used. + * + * @throws NotFoundError when no registry (or version) has the pack. + * @throws ValidationError when several registries have it and no registry was named. */ -export function findPackInRegistries(packName: string): { - matches: Array<{ entry: RegistryEntry; pack: PackSummary }>; - warnings: string[]; -} { - const warnings: string[] = []; - const matches: Array<{ entry: RegistryEntry; pack: PackSummary }> = []; - - const registries = loadRegistries(); - for (const entry of registries) { - const cacheDir = getRegistryCacheDir(entry.name); - if (!existsSync(cacheDir)) { - warnings.push( - `Registry "${entry.name}" has never been synced — skipping. Run: libscope registry sync ${entry.name}`, - ); - continue; - } +export function findRegistryPack(specifier: string, registryName?: string): ResolvedPack { + const { name, version } = parsePackSpecifier(specifier); + validatePathSegment(name, "pack name"); + if (version !== undefined) validatePathSegment(version, "version"); + + if (registryName === undefined && loadRegistries().length === 0) { + throw new NotFoundError( + `Pack "${name}" not found: no pack registries are configured. Add one with: libscope registry add <url>`, + ); + } + const { packs, warnings } = listRegistryPacks(registryName); + const matches = packs.filter((p) => p.name === name); + const [match] = matches; + if (!match) { + const where = registryName === undefined ? "any registry" : `registry "${registryName}"`; + throw new NotFoundError([`Pack "${name}" not found in ${where}.`, ...warnings].join(" ")); + } + if (matches.length > 1) { + throw new ValidationError( + `Pack "${name}" is in several registries (${matches.map((m) => m.registry).join(", ")}). ` + + "Choose one with the registry option (CLI: --registry <name>).", + ); + } - try { - const index = readIndex(cacheDir); - const found = index.find((p) => p.name === packName); - if (found) { - matches.push({ entry, pack: found }); - } - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - warnings.push(`Failed to read index for "${entry.name}": ${msg}`); - } + const chosen = version ?? match.latestVersion; + // latestVersion comes from the registry's index.json: validate it like user input. + validatePathSegment(chosen, "version"); + const dataPath = getPackDataPath(match.registry, name, chosen); + if (!existsSync(dataPath)) { + throw new NotFoundError( + `Pack "${name}@${chosen}" not found in registry "${match.registry}". ` + + `Run: libscope registry sync ${match.registry}`, + ); } + return { registry: match.registry, name, version: chosen, dataPath }; +} - return { matches, warnings }; +/** Find a pack in the registries and verify its checksum (see findRegistryPack). */ +export async function resolveRegistryPack( + specifier: string, + registryName?: string, +): Promise<ResolvedPack> { + const resolved = findRegistryPack(specifier, registryName); + await verifyResolvedPackChecksum(resolved); + return resolved; } -/** - * Read a pack manifest from the local cache. - */ +/** Read a pack manifest from the local cache. */ export function readPackManifest(registryName: string, packName: string): PackManifest | null { const manifestPath = getPackManifestPath(registryName, packName); - if (!existsSync(manifestPath)) { - // Fall back: try reading from the packs directory directly - const cacheDir = getRegistryCacheDir(registryName); - const altPath = join(cacheDir, "packs", packName, PACK_MANIFEST_FILE); - if (!existsSync(altPath)) return null; - try { - return JSON.parse(readFileSync(altPath, "utf-8")) as PackManifest; - } catch (err) { - const log = getLogger(); - log.warn( - { registryName, packName, err: err instanceof Error ? err.message : String(err) }, - "Failed to parse pack manifest (alt path)", - ); - return null; - } - } + if (!existsSync(manifestPath)) return null; try { return JSON.parse(readFileSync(manifestPath, "utf-8")) as PackManifest; } catch (err) { - const log = getLogger(); - log.warn( + getLogger().warn( { registryName, packName, err: err instanceof Error ? err.message : String(err) }, "Failed to parse pack manifest", ); @@ -115,147 +104,33 @@ export function readPackManifest(registryName: string, packName: string): PackMa } /** - * Resolve a pack from registries, handling version selection and conflicts. - * - * @param packName - Pack name (no version suffix) - * @param options - Resolution options - * @returns Resolved pack info, or null if not found - */ -export function resolvePackFromRegistries( - packName: string, - options?: { - version?: string | undefined; - registryName?: string | undefined; - conflictResolution?: ConflictResolution | undefined; - }, -): { resolved: ResolvedPack | null; conflict?: RegistryConflict; warnings: string[] } { - const log = getLogger(); - - // Validate pack name and version to prevent path traversal via malicious index.json - validatePathSegment(packName, "pack name"); - if (options?.version) { - validatePathSegment(options.version, "version"); - } - - const { matches, warnings } = findPackInRegistries(packName); - - if (matches.length === 0) { - return { resolved: null, warnings }; - } - - // Filter to specific registry if requested - let candidates = matches; - if (options?.registryName) { - candidates = matches.filter((m) => m.entry.name === options.registryName); - if (candidates.length === 0) { - warnings.push(`Pack "${packName}" not found in registry "${options.registryName}".`); - return { resolved: null, warnings }; - } - } - - // Handle conflict: multiple registries have this pack - if (candidates.length > 1) { - const conflict: RegistryConflict = { - packName, - sources: candidates.map((c) => ({ - registryName: c.entry.name, - registryUrl: c.entry.url, - version: c.pack.latestVersion, - priority: c.entry.priority, - })), - }; - - const resolution = options?.conflictResolution ?? { strategy: "priority" }; - - if (resolution.strategy === "priority") { - // Sort by priority (lower wins), pick first - candidates.sort((a, b) => a.entry.priority - b.entry.priority); - candidates = [candidates[0]!]; - log.info( - { packName, registry: candidates[0]!.entry.name }, - "Resolved pack conflict by priority", - ); - } else if (resolution.strategy === "explicit") { - const explicit = candidates.find((c) => c.entry.name === resolution.registryName); - if (!explicit) { - return { resolved: null, conflict, warnings }; - } - candidates = [explicit]; - } else { - // interactive — caller must handle the conflict - return { resolved: null, conflict, warnings }; - } - } - - const match = candidates[0]!; - const version = options?.version ?? match.pack.latestVersion; - - // Try to find the pack data file - const dataPath = getPackDataPath(match.entry.name, packName, version); - if (!existsSync(dataPath)) { - warnings.push( - `Pack "${packName}@${version}" not found in local cache for registry "${match.entry.name}". ` + - "Try syncing first: libscope registry sync", - ); - return { resolved: null, warnings }; - } - - return { - resolved: { - registryName: match.entry.name, - registryUrl: match.entry.url, - packName, - version, - dataPath, - }, - warnings, - }; -} - -/** - * Verify the checksum of a resolved pack's data file against the expected value - * stored in the pack manifest. Throws a ValidationError if the checksum does not - * match, indicating the file may have been tampered with or corrupted. - * - * Call this immediately before installing a registry-resolved pack. + * Verify the checksum of a resolved pack's data file against the value in the pack manifest. + * Throws a ValidationError if the checksum does not match (tampered or corrupted file) or the + * manifest has no checksum for the version. */ export async function verifyResolvedPackChecksum(resolved: ResolvedPack): Promise<void> { const log = getLogger(); - const manifest = readPackManifest(resolved.registryName, resolved.packName); - + const where = { registry: resolved.registry, pack: resolved.name, version: resolved.version }; + const manifest = readPackManifest(resolved.registry, resolved.name); if (!manifest) { - log.warn( - { registryName: resolved.registryName, packName: resolved.packName }, - "No pack manifest found — skipping checksum verification", - ); + log.warn(where, "No pack manifest found — skipping checksum verification"); return; } const versionEntry = manifest.versions.find((v) => v.version === resolved.version); if (!versionEntry) { - log.warn( - { - registryName: resolved.registryName, - packName: resolved.packName, - version: resolved.version, - }, - "Version entry not found in manifest — skipping checksum verification", - ); + log.warn(where, "Version entry not found in manifest — skipping checksum verification"); return; } if (!versionEntry.checksum) { throw new ValidationError( - `Pack "${resolved.packName}@${resolved.version}" in registry "${resolved.registryName}" ` + + `Pack "${resolved.name}@${resolved.version}" in registry "${resolved.registry}" ` + "has no checksum recorded. The registry may be corrupted or from an older format.", ); } // verifyChecksum throws ValidationError on mismatch await verifyChecksum(resolved.dataPath, versionEntry.checksum); - - log.info( - { registryName: resolved.registryName, packName: resolved.packName, version: resolved.version }, - "Pack checksum verified before installation", - ); + log.info(where, "Pack checksum verified before installation"); } diff --git a/src/registry/search.ts b/src/registry/search.ts index 80531b85..b5e27e9b 100644 --- a/src/registry/search.ts +++ b/src/registry/search.ts @@ -1,14 +1,20 @@ /** - * Registry search: find packs across all configured registries. + * List and search the packs of the configured registries, from their local (synced) indexes. */ import { existsSync } from "node:fs"; import { getLogger } from "../logger.js"; -import type { RegistryEntry, PackSummary, RegistrySearchResult } from "./types.js"; +import type { RegistryEntry, PackSummary, RegistryPack, RegistrySearchResult } from "./types.js"; import { getRegistryCacheDir } from "./types.js"; -import { loadRegistries } from "./config.js"; +import { loadRegistries, requireRegistry } from "./config.js"; import { readIndex } from "./git.js"; +/** Packs in registries plus warnings for registries that could not be read. */ +export interface RegistryPackList { + packs: RegistryPack[]; + warnings: string[]; +} + /** * Compute a relevance score for a pack against a query. * Higher = better match. Returns 0 for no match. @@ -50,43 +56,39 @@ function scoreMatch(pack: PackSummary, query: string): number { return score; } -/** Resolve which registries to search, returning them or adding a warning if not found. */ -function resolveRegistries( - registryName: string | undefined, - warnings: string[], -): RegistryEntry[] | null { - if (!registryName) return loadRegistries(); - const all = loadRegistries(); - const entry = all.find((r) => r.name === registryName); - if (!entry) { - warnings.push(`Registry "${registryName}" not found.`); - return null; - } - return [entry]; -} - /** Read packs from a single registry, appending warnings on failure. */ -function readRegistryPacks(entry: RegistryEntry, warnings: string[]): PackSummary[] | null { +function readRegistryPacks(entry: RegistryEntry, warnings: string[]): PackSummary[] { const cacheDir = getRegistryCacheDir(entry.name); if (!existsSync(cacheDir)) { warnings.push( `Registry "${entry.name}" has never been synced. Run: libscope registry sync ${entry.name}`, ); - return null; + return []; } try { return readIndex(cacheDir); } catch (err) { const msg = err instanceof Error ? err.message : String(err); warnings.push(`Failed to read index for "${entry.name}": ${msg}`); - getLogger().warn( - { registry: entry.name, err: msg }, - "Failed to read registry index during search", - ); - return null; + getLogger().warn({ registry: entry.name, err: msg }, "Failed to read registry index"); + return []; } } +/** + * Every pack in the local indexes of all registries, or of `registryName` only + * (NotFoundError when that registry is not configured). No network access. + */ +export function listRegistryPacks(registryName?: string): RegistryPackList { + const registries = + registryName === undefined ? loadRegistries() : [requireRegistry(registryName)]; + const warnings: string[] = []; + const packs = registries.flatMap((entry) => + readRegistryPacks(entry, warnings).map((pack) => ({ ...pack, registry: entry.name })), + ); + return { packs, warnings }; +} + /** * Search for packs across all (or a specific) registry. * Returns results sorted by relevance score (highest first). @@ -95,29 +97,12 @@ export function searchRegistries( query: string, options?: { registryName?: string | undefined }, ): { results: RegistrySearchResult[]; warnings: string[] } { - const warnings: string[] = []; - const results: RegistrySearchResult[] = []; - - const registries = resolveRegistries(options?.registryName, warnings); - if (!registries) return { results, warnings }; - - for (const entry of registries) { - const packs = readRegistryPacks(entry, warnings); - if (!packs) continue; - - for (const pack of packs) { - const score = scoreMatch(pack, query); - if (score > 0) { - results.push({ registryName: entry.name, pack, score }); - } - } - } + const { packs, warnings } = listRegistryPacks(options?.registryName); + const results = packs + .map((pack) => ({ ...pack, score: scoreMatch(pack, query) })) + .filter((r) => r.score > 0); // Sort by score descending, then by name - results.sort((a, b) => { - if (b.score !== a.score) return b.score - a.score; - return a.pack.name.localeCompare(b.pack.name); - }); - + results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); return { results, warnings }; } diff --git a/src/registry/sync.ts b/src/registry/sync.ts index 01a361fd..9e8589de 100644 --- a/src/registry/sync.ts +++ b/src/registry/sync.ts @@ -1,11 +1,14 @@ /** * Registry sync engine: keeps local caches up to date and handles offline gracefully. + * Syncs run only when asked for (`libscope registry sync`, `registry add`); reading + * registries never touches the network. */ import { existsSync, writeFileSync, readFileSync, unlinkSync, mkdirSync } from "node:fs"; import { join } from "node:path"; import { getLogger } from "../logger.js"; -import type { RegistryEntry, PackSummary, RegistrySyncStatus } from "./types.js"; +import { runConcurrent } from "../utils/async.js"; +import type { RegistryEntry, RegistrySyncStatus } from "./types.js"; import { getRegistryCacheDir } from "./types.js"; import { loadRegistries, updateRegistrySyncTime } from "./config.js"; import { cloneRegistry, fetchRegistry, readIndex, clearIndexCache } from "./git.js"; @@ -67,187 +70,88 @@ function releaseSyncLock(lockPath: string): void { } } +/** Number of packs in a registry's local index, or null when it has no readable index. */ +export function localPackCount(registryName: string): number | null { + const cacheDir = getRegistryCacheDir(registryName); + if (!existsSync(cacheDir)) return null; + try { + return readIndex(cacheDir).length; + } catch { + return null; + } +} + +/** True when the registry has a cloned local copy. */ +function hasClone(cacheDir: string): boolean { + return existsSync(join(cacheDir, ".git")); +} + /** * Sync a single registry: clone if missing, fetch if already cached. - * Returns the sync status. On failure, falls back to cached data with a warning. - * If another process is already syncing this registry, returns status "error" with - * a descriptive message so the caller can fall back to cached data. + * Never throws: on failure the status is "offline" (the cached copy is still used) or + * "error" (no cached copy, or another process is syncing this registry). */ export async function syncRegistry(entry: RegistryEntry): Promise<RegistrySyncStatus> { const log = getLogger(); const cacheDir = getRegistryCacheDir(entry.name); + const status = ( + state: RegistrySyncStatus["status"], + lastSyncedAt: string | null, + error?: string, + ): RegistrySyncStatus => ({ + registry: entry.name, + status: state, + lastSyncedAt, + ...(error === undefined ? {} : { error }), + packs: localPackCount(entry.name), + }); - const result: RegistrySyncStatus = { - registryName: entry.name, - status: "syncing", - lastSyncedAt: entry.lastSyncedAt, - }; - - // Acquire lock before touching git state const lockPath = acquireSyncLock(cacheDir); if (lockPath === null) { - // Another live process holds the lock — skip this sync - result.status = "error"; - result.error = `Registry "${entry.name}" sync is already in progress by another process. Try again shortly.`; - return result; + return status( + "error", + entry.lastSyncedAt, + `Registry "${entry.name}" sync is already in progress by another process. Try again shortly.`, + ); } try { - if (existsSync(cacheDir) && existsSync(join(cacheDir, ".git"))) { + if (hasClone(cacheDir)) { await fetchRegistry(cacheDir); } else { await cloneRegistry(entry.url, cacheDir); } - // Invalidate cached index so next readIndex() picks up fresh data clearIndexCache(cacheDir); - updateRegistrySyncTime(entry.name); - result.status = "success"; - result.lastSyncedAt = new Date().toISOString(); log.info({ registry: entry.name }, "Registry synced successfully"); + return status("success", new Date().toISOString()); } catch (err) { const message = err instanceof Error ? err.message : String(err); - - if (existsSync(cacheDir) && existsSync(join(cacheDir, ".git"))) { - // We have a cached version — fall back to it - result.status = "offline"; - result.error = message; + if (hasClone(cacheDir)) { log.warn( { registry: entry.name, err: message }, `Registry "${entry.name}" is unreachable. Using cached index from ${entry.lastSyncedAt ?? "unknown"}.`, ); - } else { - // No cache at all - result.status = "error"; - result.error = message; - log.error( - { registry: entry.name, err: message }, - `Registry "${entry.name}" has never been synced and is unreachable.`, - ); + return status("offline", entry.lastSyncedAt, message); } + log.error( + { registry: entry.name, err: message }, + `Registry "${entry.name}" has never been synced and is unreachable.`, + ); + return status("error", entry.lastSyncedAt, message); } finally { releaseSyncLock(lockPath); } - - return result; -} - -/** - * Sync a named registry. Throws if registry not found. - */ -export async function syncRegistryByName(name: string): Promise<RegistrySyncStatus> { - const registries = loadRegistries(); - const entry = registries.find((r) => r.name === name); - if (!entry) { - return { - registryName: name, - status: "error", - lastSyncedAt: null, - error: `Registry "${name}" not found. Run 'libscope registry add <url>' first.`, - }; - } - return syncRegistry(entry); } /** Maximum number of concurrent git fetch operations. */ const SYNC_CONCURRENCY = 3; -/** - * Run async tasks with a concurrency limit (worker-pool pattern). - * Returns results in the same order as the input tasks. - */ -async function runConcurrent<T>(tasks: Array<() => Promise<T>>, concurrency: number): Promise<T[]> { - const results: T[] = Array.from<T>({ length: tasks.length }); - let nextIndex = 0; - - async function worker(): Promise<void> { - while (nextIndex < tasks.length) { - const index = nextIndex++; - results[index] = await tasks[index]!(); - } - } - - const workers = Array.from({ length: Math.min(concurrency, tasks.length) }, () => worker()); - await Promise.all(workers); - return results; -} - -/** - * Sync all configured registries concurrently. Returns status for each. - */ +/** Sync all configured registries concurrently. Returns status for each. */ export async function syncAllRegistries(): Promise<RegistrySyncStatus[]> { - const registries = loadRegistries(); - if (registries.length === 0) return []; - - return runConcurrent( - registries.map((entry) => () => syncRegistry(entry)), - SYNC_CONCURRENCY, - ); -} - -/** - * Check if a registry is stale (syncInterval > 0 and time since last sync exceeds interval). - */ -export function isRegistryStale(entry: RegistryEntry): boolean { - if (entry.syncInterval <= 0) return false; - if (!entry.lastSyncedAt) return true; - - const lastSync = new Date(entry.lastSyncedAt).getTime(); - const now = Date.now(); - const intervalMs = entry.syncInterval * 1000; - return now - lastSync > intervalMs; -} - -/** - * Sync all stale registries concurrently. Intended for non-blocking startup check. - * Returns status array; errors are logged but not thrown. - */ -export async function syncStaleRegistries(): Promise<RegistrySyncStatus[]> { - const registries = loadRegistries(); - const stale = registries.filter(isRegistryStale); - if (stale.length === 0) return []; - - const log = getLogger(); - log.debug({ count: stale.length }, "Syncing stale registries concurrently"); - return runConcurrent( - stale.map((entry) => () => syncRegistry(entry)), + loadRegistries().map((entry) => () => syncRegistry(entry)), SYNC_CONCURRENCY, ); } - -/** - * Read the cached index for a registry. - * If the cache is stale, syncs first. On sync failure, uses cached data. - * Returns null with an error message if no cache exists and sync fails. - */ -export async function getRegistryIndex( - entry: RegistryEntry, -): Promise<{ packs: PackSummary[]; warning?: string }> { - const cacheDir = getRegistryCacheDir(entry.name); - - // Auto-sync if stale - if (isRegistryStale(entry) || !existsSync(cacheDir)) { - const status = await syncRegistry(entry); - if (status.status === "error") { - return { - packs: [], - warning: - status.error ?? - `Registry "${entry.name}" has never been synced and is unreachable. Run: libscope registry sync when online.`, - }; - } - if (status.status === "offline") { - const packs = readIndex(cacheDir); - return { - packs, - warning: `Registry "${entry.name}" is unreachable. Using cached index from ${entry.lastSyncedAt ?? "unknown"}.`, - }; - } - } - - // Read from cache - const packs = readIndex(cacheDir); - return { packs }; -} diff --git a/src/registry/types.ts b/src/registry/types.ts index 35cf95e8..ddf1cef5 100644 --- a/src/registry/types.ts +++ b/src/registry/types.ts @@ -21,8 +21,10 @@ import { homedir } from "node:os"; // Folder structure constants // --------------------------------------------------------------------------- -/** Root directory for all registry caches. */ -export const REGISTRIES_DIR = join(homedir(), ".libscope", "registries"); +/** Root directory for all registry caches (~/.libscope/registries, read at call time). */ +export function getRegistriesDir(): string { + return join(homedir(), ".libscope", "registries"); +} /** Name of the top-level index file in each registry. */ export const INDEX_FILE = "index.json"; @@ -44,21 +46,12 @@ export const CHECKSUM_FILE = "checksum.sha256"; export interface RegistryEntry { /** User-chosen short name (e.g. "official", "team-internal"). */ name: string; - /** Git remote URL (https or ssh). */ + /** Git remote URL (https://, ssh://, git@host:path or file://). */ url: string; - /** How often to auto-sync, in seconds. 0 = manual only. */ - syncInterval: number; - /** Priority for conflict resolution — lower wins. */ - priority: number; /** ISO-8601 timestamp of last successful sync, or null if never synced. */ lastSyncedAt: string | null; } -/** Shape of the "registries" key in ~/.libscope/config.json. */ -export interface RegistryConfigBlock { - registries: RegistryEntry[]; -} - // --------------------------------------------------------------------------- // Registry index (index.json at repo/cache root) // --------------------------------------------------------------------------- @@ -114,51 +107,32 @@ export interface PackManifest { } // --------------------------------------------------------------------------- -// Search results +// Packs listed from registries // --------------------------------------------------------------------------- -/** A pack search result, combining summary info with registry source. */ -export interface RegistrySearchResult { - /** Which registry this result came from. */ - registryName: string; - /** Pack summary from that registry's index. */ - pack: PackSummary; - /** Relevance score (higher = better match). */ - score: number; +/** A pack from a registry index, with the registry it came from. */ +export interface RegistryPack extends PackSummary { + registry: string; } -// --------------------------------------------------------------------------- -// Conflict resolution -// --------------------------------------------------------------------------- - -/** When multiple registries offer the same pack, the user must choose. */ -export interface RegistryConflict { - packName: string; - /** One entry per registry that has this pack. */ - sources: Array<{ - registryName: string; - registryUrl: string; - version: string; - priority: number; - }>; +/** A pack search result: the pack and its relevance score (higher = better match). */ +export interface RegistrySearchResult extends RegistryPack { + score: number; } -/** Resolution strategy for pack conflicts. */ -export type ConflictResolution = - | { strategy: "priority" } - | { strategy: "interactive" } - | { strategy: "explicit"; registryName: string }; - // --------------------------------------------------------------------------- // Sync state // --------------------------------------------------------------------------- -/** Status of a registry sync operation. */ +/** Result of syncing one registry. */ export interface RegistrySyncStatus { - registryName: string; - status: "syncing" | "success" | "error" | "offline"; + registry: string; + /** "offline": the fetch failed and the cached copy is used; "error": no usable copy. */ + status: "success" | "error" | "offline"; lastSyncedAt: string | null; error?: string; + /** Packs in the local index after the sync, or null when there is no readable index. */ + packs: number | null; } // --------------------------------------------------------------------------- @@ -199,30 +173,30 @@ export interface UnpublishOptions { /** Get the local cache directory for a named registry. */ export function getRegistryCacheDir(registryName: string): string { - return join(REGISTRIES_DIR, registryName); + return join(getRegistriesDir(), registryName); } /** Get the path to a registry's local index.json. */ export function getRegistryIndexPath(registryName: string): string { - return join(REGISTRIES_DIR, registryName, INDEX_FILE); + return join(getRegistriesDir(), registryName, INDEX_FILE); } /** Get the path to a pack's manifest within a registry cache. */ export function getPackManifestPath(registryName: string, packName: string): string { - return join(REGISTRIES_DIR, registryName, PACKS_DIR, packName, PACK_MANIFEST_FILE); + return join(getRegistriesDir(), registryName, PACKS_DIR, packName, PACK_MANIFEST_FILE); } /** Get the directory for a specific pack version within a registry cache. */ export function getPackVersionDir(registryName: string, packName: string, version: string): string { - return join(REGISTRIES_DIR, registryName, PACKS_DIR, packName, version); + return join(getRegistriesDir(), registryName, PACKS_DIR, packName, version); } /** Get the path to the pack data file for a specific version. */ export function getPackDataPath(registryName: string, packName: string, version: string): string { - return join(REGISTRIES_DIR, registryName, PACKS_DIR, packName, version, `${packName}.json`); + return join(getRegistriesDir(), registryName, PACKS_DIR, packName, version, `${packName}.json`); } /** Get the path to the checksum file for a specific pack version. */ export function getChecksumPath(registryName: string, packName: string, version: string): string { - return join(REGISTRIES_DIR, registryName, PACKS_DIR, packName, version, CHECKSUM_FILE); + return join(getRegistriesDir(), registryName, PACKS_DIR, packName, version, CHECKSUM_FILE); } diff --git a/src/utils/async.ts b/src/utils/async.ts new file mode 100644 index 00000000..5ffc40d2 --- /dev/null +++ b/src/utils/async.ts @@ -0,0 +1,55 @@ +/** Helpers for running async work one item at a time or with a concurrency limit. */ + +/** + * Call `fn` for each item in order. Each call starts after the previous one has resolved, + * so use this where order matters (database writes, rate-limited APIs, git commands). + * The first rejection stops the sequence and rejects the returned promise. + */ +export function forEachSequential<T>( + items: Iterable<T>, + fn: (item: T, index: number) => Promise<unknown>, +): Promise<void> { + return Array.from(items).reduce<Promise<void>>( + (previous, item, index) => + previous.then(async () => { + await fn(item, index); + }), + Promise.resolve(), + ); +} + +/** Like {@link forEachSequential}, and resolves to the results in input order. */ +export async function mapSequential<T, R>( + items: Iterable<T>, + fn: (item: T, index: number) => Promise<R>, +): Promise<R[]> { + const results: R[] = []; + await forEachSequential(items, async (item, index) => { + results.push(await fn(item, index)); + }); + return results; +} + +/** + * Run async tasks with at most `concurrency` in flight (worker-pool pattern). + * Returns results in the same order as the input tasks. + */ +export async function runConcurrent<T>( + tasks: Array<() => Promise<T>>, + concurrency: number, +): Promise<T[]> { + const results: T[] = Array.from<T>({ length: tasks.length }); + let nextIndex = 0; + + // Each worker takes the next task when its current one finishes. + const worker = async (): Promise<void> => { + if (nextIndex >= tasks.length) return; + const index = nextIndex++; + results[index] = await tasks[index]!(); + return worker(); + }; + + const workers = Array.from({ length: Math.min(concurrency, tasks.length) }, () => worker()); + await Promise.all(workers); + return results; +} diff --git a/src/utils/glob.ts b/src/utils/glob.ts new file mode 100644 index 00000000..aaf451a6 --- /dev/null +++ b/src/utils/glob.ts @@ -0,0 +1,49 @@ +const REGEX_SPECIAL = new Set([".", "+", "^", "$", "(", ")", "|", "[", "]", "\\", "{", "}"]); + +/** Regex source for the `*` at `glob[i]` (`*`, `**` or `**\/`) and how many characters it used. */ +function starSource(glob: string, i: number): [source: string, length: number] { + if (glob.charAt(i + 1) !== "*") return ["[^/]*", 1]; + return glob.charAt(i + 2) === "/" ? ["(?:.*/)?", 3] : [".*", 2]; +} + +/** Regex source for any other glob character; `{`, `,` and `}` are special only as braces. */ +function charSource(c: string, inBraces: boolean): string { + if (c === "?") return "[^/]"; + if (c === "{" && !inBraces) return "(?:"; + if (c === "}" && inBraces) return ")"; + if (c === "," && inBraces) return "|"; + return REGEX_SPECIAL.has(c) ? `\\${c}` : c; +} + +/** + * Convert a glob to an anchored RegExp that matches `/`-separated relative paths. + * + * Supports `**` (any depth; `**\/` also matches zero directories), `*` (within one + * path segment), `?` (one character), and non-nested `{a,b}` alternation. + */ +export function globToRegExp(glob: string): RegExp { + let source = ""; + let inBraces = false; + let i = 0; + + while (i < glob.length) { + const c = glob.charAt(i); + if (c === "*") { + const [part, length] = starSource(glob, i); + source += part; + i += length; + } else { + source += charSource(c, inBraces); + if (c === "{" || c === "}") inBraces = c === "{"; + i++; + } + } + if (inBraces) source += ")"; + + return new RegExp(`^${source}$`); +} + +/** Normalize a platform path to `/` separators so it can be matched by {@link globToRegExp}. */ +export function toPosixPath(path: string): string { + return path.replaceAll("\\", "/"); +} diff --git a/src/utils/markdown.ts b/src/utils/markdown.ts new file mode 100644 index 00000000..3a8ed661 --- /dev/null +++ b/src/utils/markdown.ts @@ -0,0 +1,49 @@ +/** Markdown text helpers that avoid backtracking regular expressions. */ + +const LINE_TERMINATORS = new Set(["\n", "\r", "\u2028", "\u2029"]); +const WHITESPACE = /^\s$/; + +function isLineTerminator(char: string): boolean { + return LINE_TERMINATORS.has(char); +} + +function isWhitespace(char: string): boolean { + return WHITESPACE.test(char); +} + +/** Index of the first line terminator at or after `from`, or the text length. */ +function lineEnd(text: string, from: number): number { + let end = from; + while (end < text.length && !isLineTerminator(text.charAt(end))) end++; + return end; +} + +/** + * The heading text after the "#" at `hash` (the start of a line), or undefined when it is not + * a heading. The whitespace after "#" may span lines, as `\s+` does. + */ +function headingAt(text: string, hash: number): string | undefined { + let textStart = hash + 1; + while (textStart < text.length && isWhitespace(text.charAt(textStart))) textStart++; + if (textStart === hash + 1) return undefined; + if (textStart < text.length) return text.slice(textStart, lineEnd(text, textStart)); + // Only whitespace up to the end: the heading is the last whitespace character that is not + // a line terminator, if it is not the first character after "#". + for (let i = text.length - 1; i > hash + 1; i--) { + if (!isLineTerminator(text.charAt(i))) return text.charAt(i); + } + return undefined; +} + +/** + * First markdown heading: the same text as group 1 of /^#\s+(.+)$/m (not trimmed), or + * undefined when there is none. Linear in the text length. + */ +export function firstMarkdownHeading(text: string): string | undefined { + for (let hash = text.indexOf("#"); hash !== -1; hash = text.indexOf("#", hash + 1)) { + const atLineStart = hash === 0 || isLineTerminator(text.charAt(hash - 1)); + const heading = atLineStart ? headingAt(text, hash) : undefined; + if (heading !== undefined) return heading; + } + return undefined; +} diff --git a/src/utils/strings.ts b/src/utils/strings.ts new file mode 100644 index 00000000..a41f5e5c --- /dev/null +++ b/src/utils/strings.ts @@ -0,0 +1,8 @@ +/** String helpers that avoid backtracking regular expressions. */ + +/** `value` without trailing "/" characters ("https://x.org//" -> "https://x.org"). */ +export function trimTrailingSlashes(value: string): string { + let end = value.length; + while (end > 0 && value.charAt(end - 1) === "/") end--; + return value.slice(0, end); +} diff --git a/src/web/graph-api.ts b/src/web/graph-api.ts deleted file mode 100644 index f832773b..00000000 --- a/src/web/graph-api.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { IncomingMessage, ServerResponse } from "node:http"; -import type Database from "better-sqlite3"; -import { buildKnowledgeGraph } from "../core/graph.js"; - -/** HTTP handler for graph data requests. */ -export async function handleGraphRequest( - db: Database.Database, - req: IncomingMessage, - res: ServerResponse, -): Promise<void> { - const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`); - - const rawThreshold = Number.parseFloat(url.searchParams.get("threshold") ?? "0.85"); - const rawMaxNodes = Number.parseInt(url.searchParams.get("maxNodes") ?? "200", 10); - const topic = url.searchParams.get("topic") ?? undefined; - const tag = url.searchParams.get("tag") ?? undefined; - - // Clamp parameters to safe bounds - const threshold = Number.isNaN(rawThreshold) ? 0.85 : Math.max(0, Math.min(1, rawThreshold)); - const maxNodes = Number.isNaN(rawMaxNodes) ? 200 : Math.max(1, Math.min(5000, rawMaxNodes)); - - const graph = await buildKnowledgeGraph(db, { - similarityThreshold: threshold, - maxNodes, - topicFilter: topic ?? undefined, - tagFilter: tag ?? undefined, - }); - - res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify(graph)); -} diff --git a/src/web/server.ts b/src/web/server.ts index 8ecd7669..ffd6a1e4 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -1,255 +1,194 @@ +/** + * Web dashboard server: the dashboard pages plus the small unversioned JSON API their + * scripts call. Each API route runs an operation and reshapes the result into the format + * the dashboard script expects (src/web/dashboard.ts is not changed). + */ import { createServer, type IncomingMessage, type ServerResponse, type Server } from "node:http"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../providers/embedding.js"; -import { searchDocuments } from "../core/search.js"; -import { getDocument, deleteDocument, listDocuments } from "../core/documents.js"; -import { getTopicStats } from "../core/topics.js"; +import type { z } from "zod"; +import { decodeSegment, inputFromParams, sendRequestError } from "../api/adapter.js"; +import { + corsMiddleware, + rejectForeignWrite, + sendError, + setSecurityHeaders, +} from "../api/middleware.js"; +import { listen, rejectRateLimited, type ServerApp } from "../api/server.js"; +import { + createOperationContext, + deleteDocumentOperation, + getDocumentOperation, + graphOperation, + listDocumentsOperation, + listTopicsOperation, + overviewOperation, + runOperation, + searchOperation, + type Operation, + type OperationContext, +} from "../core/operations/index.js"; import { getDashboardHtml, getGraphPageHtml } from "./dashboard.js"; -import { handleGraphRequest } from "./graph-api.js"; -import { DocumentNotFoundError } from "../errors.js"; -import { validateCountRow } from "../utils/db-validation.js"; -import { getLogger } from "../logger.js"; -import { checkRateLimit } from "../api/middleware.js"; export interface WebServerOptions { - port?: number; - host?: string; + port?: number | undefined; + host?: string | undefined; + /** Other browser origins allowed to call the dashboard API. Default: none (same origin only). */ + corsOrigins?: string[] | undefined; } -let server: Server | null = null; - -/** Start the web UI server. */ -export function startWebServer( - db: Database.Database, - provider: EmbeddingProvider, - options?: WebServerOptions, -): Promise<Server> { - const port = options?.port ?? 3377; - const host = options?.host ?? "localhost"; - - return new Promise((resolve, reject) => { - server = createServer((req, res) => { - const ip = req.socket.remoteAddress ?? "unknown"; - if (!checkRateLimit(ip)) { - res.writeHead(429, { "Content-Type": "application/json", "Retry-After": "60" }); - res.end(JSON.stringify({ error: "Too many requests" })); - return; - } - handleRequest(db, provider, req, res).catch((err) => { - getLogger().error({ err, url: req.url }, "Unhandled error in web request handler"); - if (res.headersSent) { - req.socket.destroy(); - } else { - sendJson(res, 500, { error: "Internal server error" }); - } - }); - }); +interface DashboardRoute<S extends z.ZodObject = z.ZodObject, O = unknown> { + method: "GET" | "DELETE"; + /** Path; a trailing "/:documentId" segment is passed to the operation. */ + path: string; + operation: Operation<S, O>; + /** Rename query parameters the dashboard sends under another name. */ + rename?: Record<string, string>; + /** Dashboard response shape. */ + shape: (result: O) => unknown; +} - server.on("error", reject); - server.listen(port, host, () => resolve(server!)); - }); +function route<S extends z.ZodObject, O>(r: DashboardRoute<S, O>): DashboardRoute { + return r as unknown as DashboardRoute; } -/** Gracefully shut down the web server. */ -export function stopWebServer(): Promise<void> { - return new Promise((resolve) => { - if (!server) { - resolve(); - return; +/** The JSON routes the dashboard script fetches, each served by one operation. */ +const DASHBOARD_ROUTES: DashboardRoute[] = [ + route({ + method: "GET", + path: "/api/stats", + operation: overviewOperation, + shape: ({ stats }) => ({ + documentCount: stats.totalDocuments, + topicCount: stats.totalTopics, + chunkCount: stats.totalChunks, + }), + }), + route({ + method: "GET", + path: "/api/topics", + operation: listTopicsOperation, + shape: (result) => result.items, + }), + route({ + method: "GET", + path: "/api/documents", + operation: listDocumentsOperation, + shape: (result) => result.items.map(({ documentId, ...rest }) => ({ id: documentId, ...rest })), + }), + route({ + method: "GET", + path: "/api/search", + operation: searchOperation, + rename: { q: "query" }, + shape: (result) => ({ results: result.items, totalCount: result.total }), + }), + route({ + method: "GET", + path: "/api/graph", + operation: graphOperation, + shape: (graph) => graph, + }), + route({ + method: "GET", + path: "/api/documents/:documentId", + operation: getDocumentOperation, + shape: (view) => ({ + id: view.document.documentId, + ...view.document, + content: view.content, + tags: view.tags, + }), + }), + route({ + method: "DELETE", + path: "/api/documents/:documentId", + operation: deleteDocumentOperation, + shape: () => ({ success: true }), + }), +]; + +const PAGES: Record<string, () => string> = { + "/": getDashboardHtml, + "/graph": getGraphPageHtml, +}; + +/** The route for a path, with the trailing `:documentId` value when the route has one. */ +function findRoute( + method: string, + pathname: string, +): { route: DashboardRoute; params: Record<string, string> } | undefined { + for (const r of DASHBOARD_ROUTES) { + if (r.method !== method) continue; + if (r.path === pathname) return { route: r, params: {} }; + const [prefix, param] = r.path.split("/:"); + if (param === undefined || !pathname.startsWith(`${prefix}/`)) continue; + const value = pathname.slice((prefix ?? "").length + 1); + if (value !== "" && !value.includes("/")) { + return { route: r, params: { [param]: decodeSegment(value) } }; } - server.close(() => { - server = null; - resolve(); - }); - }); + } + return undefined; } async function handleRequest( - db: Database.Database, - provider: EmbeddingProvider, + ctx: OperationContext, req: IncomingMessage, res: ServerResponse, ): Promise<void> { - const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`); - const pathname = url.pathname; + const url = new URL(req.url ?? "/", "http://localhost"); const method = req.method ?? "GET"; - - setCorsHeaders(res); - - if (method === "OPTIONS") { - res.writeHead(204); - res.end(); - return; - } - - if (method === "GET") { - const handled = await handleGetRoute(db, provider, pathname, url, res, req); - if (handled) return; - } - - const docMatch = pathname.match(/^\/api\/documents\/([^/]+)$/); - if (docMatch) { - const id = decodeURIComponent(docMatch[1]!); - const handled = handleDocumentByIdRoute(db, method, id, res); - if (handled) return; - } - - sendJson(res, 404, { error: "Not found" }); -} - -/** Handle all GET routes. Returns true if a route matched. */ -async function handleGetRoute( - db: Database.Database, - provider: EmbeddingProvider, - pathname: string, - url: URL, - res: ServerResponse, - req: IncomingMessage, -): Promise<boolean> { - if (pathname === "/") { - res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); - res.end(getDashboardHtml()); - return true; - } - - if (pathname === "/graph") { - res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); - res.end(getGraphPageHtml()); - return true; - } - - if (pathname === "/api/graph") { - await handleGraphRequest(db, req, res); - return true; - } - - if (pathname === "/api/search") { - await handleSearchRoute(db, provider, url, res); - return true; - } - - if (pathname === "/api/documents") { - handleDocumentsListRoute(db, url, res); - return true; - } - - if (pathname === "/api/topics") { - sendJson(res, 200, getTopicStats(db)); - return true; - } - - if (pathname === "/api/stats") { - handleStatsRoute(db, res); - return true; - } - - return false; -} - -/** Handle GET /api/search */ -async function handleSearchRoute( - db: Database.Database, - provider: EmbeddingProvider, - url: URL, - res: ServerResponse, -): Promise<void> { - const query = url.searchParams.get("q") ?? ""; - const rawLimit = Number.parseInt(url.searchParams.get("limit") ?? "10", 10); - const limit = Number.isNaN(rawLimit) ? 10 : Math.max(1, Math.min(100, rawLimit)); - const topic = url.searchParams.get("topic") ?? undefined; - - if (!query) { - sendJson(res, 400, { error: "Missing query parameter 'q'" }); - return; - } - - const results = await searchDocuments(db, provider, { query, limit, topic }); - sendJson(res, 200, results); -} - -/** Handle GET /api/documents */ -function handleDocumentsListRoute(db: Database.Database, url: URL, res: ServerResponse): void { - const rawLimit = Number.parseInt(url.searchParams.get("limit") ?? "50", 10); - const limit = Number.isNaN(rawLimit) ? 50 : Math.max(1, Math.min(500, rawLimit)); - const topic = url.searchParams.get("topic") ?? undefined; - const docs = listDocuments(db, { limit, topicId: topic }); - sendJson(res, 200, docs); -} - -/** Handle GET/DELETE /api/documents/:id. Returns true if method matched. */ -function handleDocumentByIdRoute( - db: Database.Database, - method: string, - id: string, - res: ServerResponse, -): boolean { - if (method === "GET") { - handleGetDocumentById(db, id, res); - return true; - } - if (method === "DELETE") { - handleDeleteDocumentById(db, id, res); - return true; - } - return false; -} - -/** Handle GET /api/documents/:id */ -function handleGetDocumentById(db: Database.Database, id: string, res: ServerResponse): void { try { - const doc = getDocument(db, id); - sendJson(res, 200, doc); - } catch (err) { - if (err instanceof DocumentNotFoundError) { - sendJson(res, 404, { error: "Document not found" }); - } else { - throw err; + const page = method === "GET" ? PAGES[url.pathname] : undefined; + if (page) { + res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + res.end(page()); + return; } - } -} - -/** Handle DELETE /api/documents/:id */ -function handleDeleteDocumentById(db: Database.Database, id: string, res: ServerResponse): void { - try { - deleteDocument(db, id); - sendJson(res, 200, { success: true }); - } catch (err) { - if (err instanceof DocumentNotFoundError) { - sendJson(res, 404, { error: "Document not found" }); - } else { - throw err; + const match = findRoute(method, url.pathname); + if (!match) { + sendError(res, 404, "NOT_FOUND", "Not found"); + return; } + const { route: r, params } = match; + const query = new URLSearchParams(); + for (const [name, value] of url.searchParams) query.append(r.rename?.[name] ?? name, value); + const input = { ...inputFromParams(r.operation, query), ...params }; + const result = await runOperation(r.operation, ctx, input); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify(r.shape(result))); + } catch (err) { + sendRequestError(res, err, { method, pathname: url.pathname }); } } -/** Handle GET /api/stats */ -function handleStatsRoute(db: Database.Database, res: ServerResponse): void { - const docCount = validateCountRow( - db.prepare("SELECT COUNT(*) AS cnt FROM documents").get(), - "document count", - ); - const topicCount = validateCountRow( - db.prepare("SELECT COUNT(*) AS cnt FROM topics").get(), - "topic count", - ); - const chunkCount = validateCountRow( - db.prepare("SELECT COUNT(*) AS cnt FROM chunks").get(), - "chunk count", - ); - sendJson(res, 200, { documentCount: docCount, topicCount, chunkCount }); -} +let server: Server | null = null; -function sendJson(res: ServerResponse, status: number, body: unknown): void { - res.writeHead(status, { "Content-Type": "application/json" }); - res.end(JSON.stringify(body)); +/** Start the web dashboard (default http://localhost:3377). */ +export async function startWebServer(app: ServerApp, options?: WebServerOptions): Promise<Server> { + const corsOrigins = options?.corsOrigins ?? []; + const ctx = createOperationContext({ ...app, surface: "api" }); + const httpServer = createServer((req, res) => { + // No Content-Security-Policy: the graph page loads d3 from a CDN. + setSecurityHeaders(res, false); + if (rejectRateLimited(req.socket.remoteAddress, res)) return; + if (corsMiddleware(req, res, corsOrigins)) return; + if (rejectForeignWrite(req, res, corsOrigins)) return; + void handleRequest(ctx, req, res); + }); + await listen(httpServer, options?.port ?? 3377, options?.host ?? "localhost"); + server = httpServer; + return httpServer; } -function setCorsHeaders(res: ServerResponse): void { - res.setHeader("Access-Control-Allow-Origin", "*"); - res.setHeader("Access-Control-Allow-Methods", "GET, DELETE, OPTIONS"); - res.setHeader("Access-Control-Allow-Headers", "Content-Type"); - res.setHeader("X-Content-Type-Options", "nosniff"); - res.setHeader("X-Frame-Options", "DENY"); - res.setHeader("X-XSS-Protection", "1; mode=block"); - res.setHeader("Referrer-Policy", "strict-origin-when-cross-origin"); +/** Gracefully shut down the web server. */ +export function stopWebServer(): Promise<void> { + return new Promise((resolve) => { + if (!server) { + resolve(); + return; + } + server.close(() => { + server = null; + resolve(); + }); + }); } diff --git a/tests/integration/lite-embed.test.ts b/tests/integration/lite-embed.test.ts index 6a48e9c3..3182422c 100644 --- a/tests/integration/lite-embed.test.ts +++ b/tests/integration/lite-embed.test.ts @@ -1,16 +1,14 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { LibScopeLite } from "../../src/lite/index.js"; +import { createLite, ValidationError, type LibScope } from "../../src/lite/index.js"; +import { initLogger } from "../../src/logger.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; /** - * Integration test: full LibScopeLite workflow. - * - * Uses a real in-memory SQLite database with MockEmbeddingProvider - * to exercise the complete pipeline: indexBatch → search → getContext → rate. + * Integration test: the libscope/lite preset end to end on a real in-memory SQLite database + * with MockEmbeddingProvider: add -> search -> ask (passthrough context) -> rate -> delete. */ -describe("LibScopeLite integration", () => { - let lite: LibScopeLite; - let provider: MockEmbeddingProvider; +describe("libscope/lite integration", () => { + let scope: LibScope; const corpus = [ { @@ -19,6 +17,7 @@ describe("LibScopeLite integration", () => { "The useState hook lets you add state to functional components. " + "Call useState with the initial state value and it returns an array with " + "the current state and a setter function. Re-renders happen when state changes.", + library: "react", }, { title: "React useEffect Hook", @@ -26,6 +25,7 @@ describe("LibScopeLite integration", () => { "useEffect runs side effects in functional components. " + "Pass a function and a dependency array. The effect re-runs when dependencies change. " + "Return a cleanup function for subscriptions or timers.", + library: "react", }, { title: "TypeScript Generics", @@ -41,132 +41,58 @@ describe("LibScopeLite integration", () => { "poll, check, and close. setTimeout and setInterval run in the timers phase. " + "setImmediate runs in the check phase, after I/O callbacks.", }, - { - title: "SQL Indexes", - content: - "Database indexes speed up queries by creating sorted data structures. " + - "B-tree indexes are the default in most databases. " + - "Composite indexes cover multiple columns and follow the leftmost prefix rule.", - }, ]; beforeAll(async () => { - provider = new MockEmbeddingProvider(); - lite = new LibScopeLite({ dbPath: ":memory:", provider }); - await lite.indexBatch(corpus, { concurrency: 2 }); + initLogger("silent"); + scope = createLite({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + config: { llm: { provider: "passthrough" } }, + }); + await Promise.all(corpus.map((doc) => scope.add(doc))); }); afterAll(() => { - lite.close(); + scope.close(); }); - describe("indexBatch → search", () => { - it("should find indexed documents via search", async () => { - const results = await lite.search("React hooks"); - expect(results.length).toBeGreaterThan(0); - }); - - it("should return results with all expected fields", async () => { - const results = await lite.search("generics"); - expect(results.length).toBeGreaterThan(0); - - const r = results[0]!; - expect(typeof r.docId).toBe("string"); - expect(typeof r.chunkId).toBe("string"); - expect(typeof r.title).toBe("string"); - expect(typeof r.content).toBe("string"); - expect(typeof r.score).toBe("number"); - expect(r.score).toBeGreaterThan(0); - }); - - it("should respect the limit option", async () => { - const results = await lite.search("Node.js", { limit: 2 }); - expect(results.length).toBeLessThanOrEqual(2); - }); - - it("should return results for different queries", async () => { - const r1 = await lite.search("React useState"); - const r2 = await lite.search("SQL database index"); - - expect(r1.length).toBeGreaterThan(0); - expect(r2.length).toBeGreaterThan(0); - }); + it("search returns chunk results with document IDs", async () => { + const { items, total } = await scope.search({ query: "generics", limit: 2 }); + expect(items.length).toBeGreaterThan(0); + expect(items.length).toBeLessThanOrEqual(2); + expect(total).toBeGreaterThanOrEqual(items.length); + const r = items[0]!; + expect(typeof r.documentId).toBe("string"); + expect(typeof r.chunkId).toBe("string"); + expect(typeof r.score).toBe("number"); }); - describe("getContext", () => { - it("should return a context prompt string containing relevant content", async () => { - const context = await lite.getContext("How does the Node.js event loop work?"); - expect(typeof context).toBe("string"); - expect(context.length).toBeGreaterThan(0); - }); - - it("should include question in context", async () => { - const context = await lite.getContext("What are TypeScript generics?"); - // The context prompt typically includes the question - expect(context).toContain("TypeScript generics"); - }); + it("filters by library", async () => { + const { items } = await scope.search({ query: "functional components", library: "react" }); + expect(items.length).toBeGreaterThan(0); + expect(items.every((r) => r.library === "react")).toBe(true); }); - describe("rate", () => { - it("should rate a document found via search", async () => { - const results = await lite.search("React hooks"); - expect(results.length).toBeGreaterThan(0); - - const docId = results[0]!.docId; - // Should not throw - lite.rate(docId, 5); - lite.rate(docId, 3); - }); - - it("should reject invalid ratings", async () => { - // We need a valid doc ID first - const rateInvalid = async (): Promise<void> => { - const results = await lite.search("React"); - const docId = results[0]!.docId; - lite.rate(docId, 0); // 0 is out of range - }; - await expect(rateInvalid()).rejects.toThrow(); - }); + it("ask in passthrough mode returns the context prompt and sources", async () => { + const result = await scope.ask("How does the Node.js event loop work?"); + expect(result.mode).toBe("context"); + if (result.mode === "context") { + expect(result.contextPrompt).toContain("event loop"); + expect(result.sources.length).toBeGreaterThan(0); + } }); - describe("full pipeline: index → search → getContext → rate", () => { - it("should execute the complete workflow end-to-end", async () => { - // 1. Index additional docs - const extraLite = new LibScopeLite({ - dbPath: ":memory:", - provider: new MockEmbeddingProvider(), - }); - await extraLite.index([ - { - title: "Docker Basics", - content: - "Docker containers package applications with their dependencies. " + - "Images are built from Dockerfiles. Containers run as isolated processes.", - library: "docker", - }, - { - title: "Kubernetes Pods", - content: - "Kubernetes pods are the smallest deployable units. " + - "A pod can contain one or more containers sharing network and storage.", - library: "kubernetes", - }, - ]); - - // 2. Search - const searchResults = await extraLite.search("Docker containers"); - expect(searchResults.length).toBeGreaterThan(0); - expect(searchResults[0]!.title).toBeDefined(); - - // 3. Get context - const context = await extraLite.getContext("How does Docker work?"); - expect(context.length).toBeGreaterThan(0); - - // 4. Rate - const docId = searchResults[0]!.docId; - extraLite.rate(docId, 4); - - extraLite.close(); - }); + it("rates, rejects an invalid rating, and deletes", async () => { + const { items } = await scope.search("React hooks"); + const documentId = items[0]!.documentId; + await scope.docs.rate({ documentId, rating: 5 }); + await expect(scope.docs.rate({ documentId, rating: 0 })).rejects.toBeInstanceOf( + ValidationError, + ); + expect((await scope.docs.get({ documentId })).ratings.totalRatings).toBe(1); + + await scope.docs.delete({ documentId }); + expect((await scope.docs.list()).total).toBe(corpus.length - 1); }); }); diff --git a/tests/integration/parsed-file-import.test.ts b/tests/integration/parsed-file-import.test.ts new file mode 100644 index 00000000..cbc20caf --- /dev/null +++ b/tests/integration/parsed-file-import.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import PizZip from "pizzip"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { ingest } from "../../src/core/ingest.js"; +import { FileWatcher, readWatchedFile } from "../../src/core/watcher.js"; +import { initLogger } from "../../src/logger.js"; + +const DOCX_TEXT = "Quarterly revenue grew in the northern region"; + +/** Build a minimal but valid .docx (Office Open XML) containing one paragraph. */ +function buildDocx(text: string): Buffer { + const zip = new PizZip(); + zip.file( + "[Content_Types].xml", + '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' + + '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">' + + '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>' + + '<Default Extension="xml" ContentType="application/xml"/>' + + '<Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>' + + "</Types>", + ); + zip.file( + "_rels/.rels", + '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' + + '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">' + + '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>' + + "</Relationships>", + ); + zip.file( + "word/document.xml", + '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' + + '<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">' + + `<w:body><w:p><w:r><w:t>${text}</w:t></w:r></w:p></w:body>` + + "</w:document>", + ); + return zip.generate({ type: "nodebuffer" }); +} + +function documentContent(db: Database.Database, id: string): { content: string; source: string } { + return db + .prepare("SELECT content, source_type AS source FROM documents WHERE id = ?") + .get(id) as { content: string; source: string }; +} + +describe("integration: directory ingest and watch use the parser registry", () => { + let db: Database.Database; + let provider: MockEmbeddingProvider; + let dir: string; + let docxPath: string; + + beforeEach(() => { + initLogger("silent"); + db = createTestDbWithVec(); + provider = new MockEmbeddingProvider(); + dir = mkdtempSync(join(tmpdir(), "libscope-parsed-import-")); + docxPath = join(dir, "report.docx"); + writeFileSync(docxPath, buildDocx(DOCX_TEXT)); + }); + + afterEach(() => { + db.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + it("directory ingest indexes parsed DOCX text and skips unsupported files", async () => { + const pngPath = join(dir, "logo.png"); + writeFileSync(pngPath, Buffer.from([0x89, 0x50, 0x4e, 0x47])); + + const result = await ingest( + { db, provider }, + { source: dir, kind: "directory", sourceType: "model-generated" }, + ); + + expect(result.documents).toHaveLength(1); + expect(result.errors).toEqual([]); + expect(result.skipped).toEqual([{ source: pngPath, reason: "unsupported file format" }]); + + const doc = documentContent(db, result.documents[0]!.documentId); + expect(doc.content).toContain(DOCX_TEXT); + expect(doc.content).not.toContain("word/document.xml"); + expect(doc.source).toBe("model-generated"); + }); + + it("readWatchedFile parses binary formats and reads .rst as text", async () => { + const rstPath = join(dir, "guide.rst"); + writeFileSync(rstPath, "Title\n=====\n\nSome reStructuredText."); + + expect((await readWatchedFile(docxPath))?.trim()).toBe(DOCX_TEXT); + expect(await readWatchedFile(rstPath)).toContain("Some reStructuredText."); + expect(await readWatchedFile(join(dir, "logo.png"))).toBeNull(); + }); + + it("FileWatcher indexes the parsed DOCX text for a changed file", async () => { + const indexed: string[] = []; + const watcher = new FileWatcher(db, provider, { + directory: dir, + onIndex: (path): void => { + indexed.push(path); + }, + }); + + // Drive the change handler directly instead of waiting on fs.watch events. + await (watcher as unknown as { processFile(p: string): Promise<void> }).processFile(docxPath); + + expect(indexed).toEqual([docxPath]); + const row = db.prepare("SELECT content FROM documents WHERE url = ?").get(docxPath) as { + content: string; + }; + expect(row.content).toContain(DOCX_TEXT); + expect(row.content).not.toContain("word/document.xml"); + }); +}); diff --git a/tests/integration/registry/registry-conflict.test.ts b/tests/integration/registry/registry-conflict.test.ts index f682cb16..af89ad24 100644 --- a/tests/integration/registry/registry-conflict.test.ts +++ b/tests/integration/registry/registry-conflict.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from "node:os"; import { randomUUID, createHash } from "node:crypto"; import { execSync } from "node:child_process"; import { initLogger } from "../../../src/logger.js"; +import { NotFoundError } from "../../../src/errors.js"; import type { RegistryEntry, PackSummary } from "../../../src/registry/types.js"; const gitEnv = { @@ -29,10 +30,13 @@ vi.mock("node:os", async (importOriginal) => { const { loadRegistries, saveRegistries, getRegistry } = await import("../../../src/registry/config.js"); const { syncRegistry } = await import("../../../src/registry/sync.js"); -const { resolvePackFromRegistries } = await import("../../../src/registry/resolve.js"); +const { findRegistryPack } = await import("../../../src/registry/resolve.js"); +const { installPackOperation, listPacksOperation, runOperation } = + await import("../../../src/core/operations/index.js"); +const { makeContext } = await import("../../unit/operations/helpers.js"); -function makeEntry(name: string, url: string, priority = 1): RegistryEntry { - return { name, url, syncInterval: 3600, priority, lastSyncedAt: null }; +function makeEntry(name: string, url: string): RegistryEntry { + return { name, url, lastSyncedAt: null }; } function addTestRegistry(entry: RegistryEntry): void { @@ -112,183 +116,69 @@ describe("integration: registry conflict resolution", () => { rmSync(tempDir, { recursive: true, force: true }); }); - it("should detect conflict when two registries have the same pack name", async () => { - const sharedPack: PackSummary = { - name: "shared-pack", - description: "Shared", + function pack(name: string, author = "author"): PackSummary { + return { + name, + description: `The ${name} pack`, tags: [], latestVersion: "1.0.0", - author: "author", + author, updatedAt: "2026-01-01", }; + } - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [{ ...sharedPack, author: "other-author" }]); - - addTestRegistry(makeEntry("reg1", repo1, 2)); - addTestRegistry(makeEntry("reg2", repo2, 1)); - - await syncRegistry(getRegistry("reg1")!); - await syncRegistry(getRegistry("reg2")!); - - // With interactive resolution, should get conflict back - const { resolved, conflict } = resolvePackFromRegistries("shared-pack", { - conflictResolution: { strategy: "interactive" }, - }); - - expect(resolved).toBeNull(); - expect(conflict).toBeDefined(); - expect(conflict!.sources).toHaveLength(2); - expect(conflict!.sources.map((s) => s.registryName).sort((a, b) => a.localeCompare(b))).toEqual( - ["reg1", "reg2"], - ); - }); - - it("should resolve conflict with explicit --registry flag", async () => { - const sharedPack: PackSummary = { - name: "shared-pack", - description: "Shared", - tags: [], - latestVersion: "1.0.0", - author: "author-1", - updatedAt: "2026-01-01", - }; - - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [{ ...sharedPack, author: "author-2" }]); - - addTestRegistry(makeEntry("reg1", repo1)); - addTestRegistry(makeEntry("reg2", repo2)); - await syncRegistry(getRegistry("reg1")!); - await syncRegistry(getRegistry("reg2")!); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - registryName: "reg1", - }); - - expect(resolved).not.toBeNull(); - expect(resolved!.registryName).toBe("reg1"); - }); - - it("should resolve conflict by priority (lower wins)", async () => { - const sharedPack: PackSummary = { - name: "priority-pack", - description: "Priority test", - tags: [], - latestVersion: "1.0.0", - author: "author", - updatedAt: "2026-01-01", - }; - - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [sharedPack]); - - addTestRegistry(makeEntry("high-priority", repo1, 10)); - addTestRegistry(makeEntry("low-priority", repo2, 1)); - await syncRegistry(getRegistry("high-priority")!); - await syncRegistry(getRegistry("low-priority")!); - - const { resolved } = resolvePackFromRegistries("priority-pack", { - conflictResolution: { strategy: "priority" }, - }); - - expect(resolved).not.toBeNull(); - expect(resolved!.registryName).toBe("low-priority"); - }); - - it("should not conflict when packs have different names", async () => { - const repo1 = createBareRepoWithPacks(tempDir, [ - { - name: "pack-a", - description: "Pack A", - tags: [], - latestVersion: "1.0.0", - author: "a", - updatedAt: "2026-01-01", - }, - ]); - const repo2 = createBareRepoWithPacks(tempDir, [ - { - name: "pack-b", - description: "Pack B", - tags: [], - latestVersion: "1.0.0", - author: "b", - updatedAt: "2026-01-01", - }, - ]); - - addTestRegistry(makeEntry("no-conflict-1", repo1)); - addTestRegistry(makeEntry("no-conflict-2", repo2)); - await syncRegistry(getRegistry("no-conflict-1")!); - await syncRegistry(getRegistry("no-conflict-2")!); - - const { resolved: r1, conflict: c1 } = resolvePackFromRegistries("pack-a"); - expect(r1).not.toBeNull(); - expect(c1).toBeUndefined(); + async function addSynced(name: string, packs: PackSummary[]): Promise<void> { + addTestRegistry(makeEntry(name, createBareRepoWithPacks(tempDir, packs))); + expect((await syncRegistry(getRegistry(name)!)).status).toBe("success"); + } - const { resolved: r2, conflict: c2 } = resolvePackFromRegistries("pack-b"); - expect(r2).not.toBeNull(); - expect(c2).toBeUndefined(); + it("refuses an ambiguous pack name and installs from the named registry", async () => { + await addSynced("reg1", [pack("shared-pack")]); + await addSynced("reg2", [pack("shared-pack", "other-author")]); + const t = makeContext(); + try { + await expect( + runOperation(installPackOperation, t.ctx, { pack: "shared-pack" }), + ).rejects.toThrow(/several registries \(reg1, reg2\)/); + + const result = await runOperation(installPackOperation, t.ctx, { + pack: "shared-pack@1.0.0", + registry: "reg2", + }); + expect(result).toMatchObject({ packName: "shared-pack", documentsInstalled: 1 }); + const row = t.db.prepare("SELECT content FROM documents").get() as { content: string }; + expect(row.content).toContain("other-author"); + } finally { + t.db.close(); + } }); - it("should handle conflict with three registries", async () => { - const sharedPack: PackSummary = { - name: "triple-pack", - description: "Three-way conflict", - tags: [], - latestVersion: "1.0.0", - author: "author", - updatedAt: "2026-01-01", - }; - - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo3 = createBareRepoWithPacks(tempDir, [sharedPack]); - - addTestRegistry(makeEntry("triple-1", repo1)); - addTestRegistry(makeEntry("triple-2", repo2)); - addTestRegistry(makeEntry("triple-3", repo3)); - await syncRegistry(getRegistry("triple-1")!); - await syncRegistry(getRegistry("triple-2")!); - await syncRegistry(getRegistry("triple-3")!); - - const { conflict } = resolvePackFromRegistries("triple-pack", { - conflictResolution: { strategy: "interactive" }, - }); - - expect(conflict).toBeDefined(); - expect(conflict!.sources).toHaveLength(3); + it("resolves packs with different names without a registry", async () => { + await addSynced("one", [pack("alpha")]); + await addSynced("two", [pack("beta")]); + expect(findRegistryPack("alpha").registry).toBe("one"); + expect(findRegistryPack("beta").registry).toBe("two"); + expect(() => findRegistryPack("gamma")).toThrow(NotFoundError); }); - it("should list all conflicting registries in conflict object", async () => { - const sharedPack: PackSummary = { - name: "info-pack", - description: "Info test", - tags: [], - latestVersion: "1.0.0", - author: "author", - updatedAt: "2026-01-01", - }; - - const repo1 = createBareRepoWithPacks(tempDir, [sharedPack]); - const repo2 = createBareRepoWithPacks(tempDir, [sharedPack]); - - addTestRegistry(makeEntry("info-reg1", repo1, 2)); - addTestRegistry(makeEntry("info-reg2", repo2, 1)); - await syncRegistry(getRegistry("info-reg1")!); - await syncRegistry(getRegistry("info-reg2")!); - - const { conflict } = resolvePackFromRegistries("info-pack", { - conflictResolution: { strategy: "interactive" }, - }); - - expect(conflict!.packName).toBe("info-pack"); - for (const source of conflict!.sources) { - expect(source.registryName).toBeTruthy(); - expect(source.registryUrl).toBeTruthy(); - expect(source.version).toBe("1.0.0"); - expect(typeof source.priority).toBe("number"); + it("lists the packs available in the synced registries", async () => { + await addSynced("one", [pack("alpha")]); + await addSynced("two", [pack("beta"), pack("alpha")]); + const t = makeContext(); + try { + const all = await runOperation(listPacksOperation, t.ctx, { available: true }); + expect(all.items.map((p) => ("registry" in p ? `${p.registry}/${p.name}` : p.name))).toEqual([ + "one/alpha", + "two/beta", + "two/alpha", + ]); + const two = await runOperation(listPacksOperation, t.ctx, { + available: true, + registry: "two", + }); + expect(two.items).toHaveLength(2); + } finally { + t.db.close(); } }); }); diff --git a/tests/integration/registry/registry-lifecycle.test.ts b/tests/integration/registry/registry-lifecycle.test.ts index f510723b..adc837ff 100644 --- a/tests/integration/registry/registry-lifecycle.test.ts +++ b/tests/integration/registry/registry-lifecycle.test.ts @@ -37,8 +37,6 @@ function makeEntry(name: string, url: string): RegistryEntry { return { name, url, - syncInterval: 3600, - priority: 1, lastSyncedAt: null, }; } @@ -188,7 +186,7 @@ describe("integration: registry lifecycle", () => { const { results } = searchRegistries("react"); expect(results.length).toBeGreaterThan(0); - expect(results[0]!.pack.name).toBe("react-docs"); + expect(results[0]!.name).toBe("react-docs"); }); it("should remove a registry and clean up cache", async () => { @@ -284,8 +282,8 @@ describe("integration: registry lifecycle", () => { // Search across both const { results } = searchRegistries("pack"); - expect(results.length).toBe(2); - expect(results.map((r) => r.pack.name).sort((a, b) => a.localeCompare(b))).toEqual([ + expect(results).toHaveLength(2); + expect(results.map((r) => r.name).sort((a, b) => a.localeCompare(b))).toEqual([ "pack-from-reg1", "pack-from-reg2", ]); diff --git a/tests/integration/registry/registry-offline.test.ts b/tests/integration/registry/registry-offline.test.ts index 30889e93..8ad55cf1 100644 --- a/tests/integration/registry/registry-offline.test.ts +++ b/tests/integration/registry/registry-offline.test.ts @@ -28,12 +28,12 @@ vi.mock("node:os", async (importOriginal) => { const { loadRegistries, saveRegistries, getRegistry } = await import("../../../src/registry/config.js"); -const { syncRegistry, getRegistryIndex } = await import("../../../src/registry/sync.js"); -const { searchRegistries } = await import("../../../src/registry/search.js"); +const { syncRegistry } = await import("../../../src/registry/sync.js"); +const { searchRegistries, listRegistryPacks } = await import("../../../src/registry/search.js"); const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); function makeEntry(name: string, url: string): RegistryEntry { - return { name, url, syncInterval: 3600, priority: 1, lastSyncedAt: null }; + return { name, url, lastSyncedAt: null }; } function addTestRegistry(entry: RegistryEntry): void { @@ -120,7 +120,7 @@ describe("integration: registry offline / unreachable remote", () => { const entry = getRegistry("named-error")!; const status = await syncRegistry(entry); - expect(status.registryName).toBe("named-error"); + expect(status.registry).toBe("named-error"); expect(status.status).toBe("error"); }); @@ -147,7 +147,7 @@ describe("integration: registry offline / unreachable remote", () => { // Search should still return results from cache const { results } = searchRegistries("stale"); expect(results.length).toBeGreaterThan(0); - expect(results[0]!.pack.name).toBe("stale-searchable"); + expect(results[0]!.name).toBe("stale-searchable"); }); it("should fail clearly when no cache exists and remote is unreachable", async () => { @@ -162,13 +162,12 @@ describe("integration: registry offline / unreachable remote", () => { expect(existsSync(getRegistryCacheDir("no-cache"))).toBe(false); }); - it("should return error with getRegistryIndex when never synced and unreachable", async () => { + it("warns about a registry that was never synced when listing packs", () => { addTestRegistry(makeEntry("never-synced", join(tempDir, "ghost.git"))); - const entry = getRegistry("never-synced")!; - const { packs, warning } = await getRegistryIndex(entry); + const { packs, warnings } = listRegistryPacks(); expect(packs).toEqual([]); - expect(warning).toBeTruthy(); + expect(warnings[0]).toContain("never been synced"); }); }); diff --git a/tests/integration/registry/registry-publish.test.ts b/tests/integration/registry/registry-publish.test.ts index 45baf361..a47931a1 100644 --- a/tests/integration/registry/registry-publish.test.ts +++ b/tests/integration/registry/registry-publish.test.ts @@ -29,12 +29,13 @@ vi.mock("node:os", async (importOriginal) => { const { loadRegistries, saveRegistries, getRegistry } = await import("../../../src/registry/config.js"); const { syncRegistry } = await import("../../../src/registry/sync.js"); -const { publishPack, unpublishPack } = await import("../../../src/registry/publish.js"); +const { publishPack, publishPackToBranch, unpublishPack } = + await import("../../../src/registry/publish.js"); const { verifyChecksum } = await import("../../../src/registry/checksum.js"); const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); function makeEntry(name: string, url: string): RegistryEntry { - return { name, url, syncInterval: 3600, priority: 1, lastSyncedAt: null }; + return { name, url, lastSyncedAt: null }; } function addTestRegistry(entry: RegistryEntry): void { @@ -166,6 +167,27 @@ describe("integration: registry publish", () => { expect(await verifyChecksum(publishedFile, result.checksum)).toBe(true); }); + it("should submit a pack on a pushed feature branch and return to the base branch", async () => { + const bareRepo = createBareRepo(tempDir); + addTestRegistry(makeEntry("submit-reg", bareRepo)); + await syncRegistry(getRegistry("submit-reg")!); + const cacheDir = getRegistryCacheDir("submit-reg"); + const baseBranch = execSync("git rev-parse --abbrev-ref HEAD", { cwd: cacheDir }).toString(); + + const packFile = createPackFile(tempDir, "submitted-pack"); + const result = await publishPackToBranch({ + registryName: "submit-reg", + packFilePath: packFile, + version: "1.0.0", + }); + + expect(result.branch).toBe("feature/add-submitted-pack"); + const remoteBranches = execSync("git branch --list", { cwd: bareRepo }).toString(); + expect(remoteBranches).toContain("feature/add-submitted-pack"); + const current = execSync("git rev-parse --abbrev-ref HEAD", { cwd: cacheDir }).toString(); + expect(current).toBe(baseBranch); + }); + it("should unpublish a pack version", async () => { const bareRepo = createBareRepo(tempDir); addTestRegistry(makeEntry("unpub-reg", bareRepo)); @@ -331,3 +353,112 @@ describe("integration: registry publish", () => { ).rejects.toThrow(/not found/); }); }); + +describe("integration: registry publish to a feature branch (--submit)", () => { + let tempDir: string; + let savedGlobalConfig: string | undefined; + + /** Run git in `cwd` and return its trimmed output. */ + function gitOut(args: string, cwd: string): string { + return execSync(`git ${args}`, { cwd, stdio: "pipe" }).toString().trim(); + } + + beforeEach(() => { + initLogger("silent"); + tempDir = mkdtempSync(join(tmpdir(), "libscope-submit-")); + tempHome = join(tempDir, "home"); + mkdirSync(tempHome, { recursive: true }); + // Default git settings: no push.autoSetupRemote or other options from the user's config. + savedGlobalConfig = process.env["GIT_CONFIG_GLOBAL"]; + const emptyConfig = join(tempDir, "gitconfig"); + writeFileSync(emptyConfig, "", "utf-8"); + process.env["GIT_CONFIG_GLOBAL"] = emptyConfig; + }); + + afterEach(() => { + if (savedGlobalConfig === undefined) delete process.env["GIT_CONFIG_GLOBAL"]; + else process.env["GIT_CONFIG_GLOBAL"] = savedGlobalConfig; + rmSync(tempDir, { recursive: true, force: true }); + }); + + async function setUpRegistry(name: string): Promise<{ bareRepo: string; cacheDir: string }> { + const bareRepo = createBareRepo(tempDir); + addTestRegistry(makeEntry(name, bareRepo)); + await syncRegistry(getRegistry(name)!); + return { bareRepo, cacheDir: getRegistryCacheDir(name) }; + } + + it("pushes the pack to feature/add-<pack> and leaves the main branch unchanged", async () => { + const { bareRepo, cacheDir } = await setUpRegistry("submit-reg"); + const mainBranch = gitOut("rev-parse --abbrev-ref HEAD", cacheDir); + const mainBefore = gitOut(`rev-parse ${mainBranch}`, bareRepo); + + const result = await publishPackToBranch({ + registryName: "submit-reg", + packFilePath: createPackFile(tempDir, "sub-pack", "1.2.0"), + }); + + expect(result).toMatchObject({ + packName: "sub-pack", + version: "1.2.0", + branch: "feature/add-sub-pack", + }); + // The remote has the branch with the pack files and the index entry. + const branch = "feature/add-sub-pack"; + expect(gitOut(`log -1 --format=%s ${branch}`, bareRepo)).toBe("feat: add sub-pack@1.2.0"); + expect(gitOut(`ls-tree -r --name-only ${branch}`, bareRepo).split("\n")).toEqual( + expect.arrayContaining([ + "packs/sub-pack/pack.json", + "packs/sub-pack/1.2.0/sub-pack.json", + "packs/sub-pack/1.2.0/checksum.sha256", + ]), + ); + const index = JSON.parse(gitOut(`show ${branch}:index.json`, bareRepo)) as PackSummary[]; + expect(index.map((p) => p.name)).toEqual(["sub-pack"]); + // The main branch did not move, and the local clone is back on it, clean. + expect(gitOut(`rev-parse ${mainBranch}`, bareRepo)).toBe(mainBefore); + expect(gitOut("rev-parse --abbrev-ref HEAD", cacheDir)).toBe(mainBranch); + expect(gitOut("status --porcelain", cacheDir)).toBe(""); + expect(gitOut("branch --list feature/*", cacheDir)).toBe(""); + expect(existsSync(join(cacheDir, "packs", "sub-pack"))).toBe(false); + }); + + it("a later normal publish goes to the main branch", async () => { + const { bareRepo, cacheDir } = await setUpRegistry("submit-then-main"); + const mainBranch = gitOut("rev-parse --abbrev-ref HEAD", cacheDir); + await publishPackToBranch({ + registryName: "submit-then-main", + packFilePath: createPackFile(tempDir, "first-pack"), + }); + + await publishPack({ + registryName: "submit-then-main", + packFilePath: createPackFile(tempDir, "second-pack"), + }); + + expect(gitOut(`log -1 --format=%s ${mainBranch}`, bareRepo)).toBe("publish: second-pack@1.0.0"); + expect(gitOut("log -1 --format=%s feature/add-first-pack", bareRepo)).toBe( + "feat: add first-pack@1.0.0", + ); + }); + + it("returns the clone to its branch when the publish fails", async () => { + const { bareRepo, cacheDir } = await setUpRegistry("submit-fail"); + const mainBranch = gitOut("rev-parse --abbrev-ref HEAD", cacheDir); + const packFile = createPackFile(tempDir, "dup-pack"); + await publishPack({ registryName: "submit-fail", packFilePath: packFile, version: "1.0.0" }); + + await expect( + publishPackToBranch({ + registryName: "submit-fail", + packFilePath: packFile, + version: "1.0.0", + }), + ).rejects.toThrow(/already exists/); + + expect(gitOut("rev-parse --abbrev-ref HEAD", cacheDir)).toBe(mainBranch); + expect(gitOut("status --porcelain", cacheDir)).toBe(""); + expect(gitOut("branch --list feature/*", cacheDir)).toBe(""); + expect(gitOut("branch --list feature/*", bareRepo)).toBe(""); + }); +}); diff --git a/tests/unit/api-routes.test.ts b/tests/unit/api-routes.test.ts new file mode 100644 index 00000000..f4c36168 --- /dev/null +++ b/tests/unit/api-routes.test.ts @@ -0,0 +1,563 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from "vitest"; +import { createServer, request, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { testConfig } from "./operations/helpers.js"; +import { initLogger } from "../../src/logger.js"; + +// Saved connections live under a temp HOME, never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-api-routes-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); +// Webhook URLs resolve to a public address without network access. +vi.mock("node:dns", async (importOriginal) => { + const actual: typeof import("node:dns") = await importOriginal(); + return { + ...actual, + promises: { + ...actual.promises, + resolve4: vi.fn().mockResolvedValue(["93.184.216.34"]), + resolve6: vi.fn().mockResolvedValue([]), + }, + }; +}); + +const { API_ROUTES, handleRequest } = await import("../../src/api/routes.js"); +const { createOperationContext } = await import("../../src/core/operations/index.js"); +const { indexDocument } = await import("../../src/core/indexing.js"); +const { updateDocument } = await import("../../src/core/documents.js"); +const { createTopic } = await import("../../src/core/topics.js"); +const { createLink } = await import("../../src/core/links.js"); +const { createSavedSearch } = await import("../../src/core/saved-searches.js"); +const { createWebhook } = await import("../../src/core/webhooks.js"); +const { taskRegistry } = await import("../../src/core/tasks.js"); +const { saveConnectorSettings } = await import("../../src/connectors/saved-config.js"); + +initLogger("silent"); + +interface Fixtures { + docA: string; + docB: string; + linkId: string; + webhookId: string; + taskId: string; +} + +interface Case { + path: string; + query?: Record<string, string>; + body?: unknown; +} + +interface RouteCases { + ok: (f: Fixtures) => Case; + /** Expected success status when not 200 (202 for background tasks). */ + okStatus?: number; + /** A request that fails validation (400). Absent when no input can be invalid. */ + bad?: (f: Fixtures) => Case; + /** A request naming a missing resource (404). */ + missing?: (f: Fixtures) => Case; +} + +const V = "/api/v1"; +const doc = (f: Fixtures, rest = ""): string => `${V}/documents/${f.docA}${rest}`; + +/** One entry per route; the test below fails when a route has no entry. */ +const CASES: Record<string, RouteCases> = { + "GET /openapi.json": { ok: () => ({ path: "/openapi.json" }) }, + "GET /api/v1/health": { ok: () => ({ path: `${V}/health` }) }, + + "POST /api/v1/documents": { + ok: () => ({ path: `${V}/documents`, body: { title: "New", content: "New body text" } }), + okStatus: 202, + bad: () => ({ path: `${V}/documents`, body: { kind: "bogus" } }), + }, + "GET /api/v1/documents/:documentId": { + ok: (f) => ({ path: doc(f), query: { maxLength: "5" } }), + bad: (f) => ({ path: doc(f), query: { offset: "-1" } }), + missing: () => ({ path: `${V}/documents/nope` }), + }, + "GET /api/v1/documents": { + ok: () => ({ path: `${V}/documents`, query: { topic: "Guides", limit: "5" } }), + bad: () => ({ path: `${V}/documents`, query: { limit: "0" } }), + }, + "PATCH /api/v1/documents/:documentId": { + ok: (f) => ({ path: doc(f), body: { title: "Renamed", tags: ["a", "b"] } }), + bad: (f) => ({ path: doc(f), body: {} }), + missing: () => ({ path: `${V}/documents/nope`, body: { title: "x" } }), + }, + "DELETE /api/v1/documents/:documentId": { + ok: (f) => ({ path: doc(f) }), + missing: () => ({ path: `${V}/documents/nope` }), + }, + "POST /api/v1/documents/:documentId/ratings": { + ok: (f) => ({ path: doc(f, "/ratings"), body: { rating: 5 } }), + bad: (f) => ({ path: doc(f, "/ratings"), body: { rating: 9 } }), + missing: () => ({ path: `${V}/documents/nope/ratings`, body: { rating: 3 } }), + }, + "GET /api/v1/documents/:documentId/versions": { + ok: (f) => ({ path: doc(f, "/versions") }), + missing: () => ({ path: `${V}/documents/nope/versions` }), + }, + "POST /api/v1/documents/:documentId/rollback": { + ok: (f) => ({ path: doc(f, "/rollback"), body: { version: 1 } }), + bad: (f) => ({ path: doc(f, "/rollback"), body: { version: 0 } }), + missing: () => ({ path: `${V}/documents/nope/rollback`, body: { version: 1 } }), + }, + + "GET /api/v1/search": { + ok: () => ({ path: `${V}/search`, query: { query: "typescript", tags: "intro,guide" } }), + bad: () => ({ path: `${V}/search` }), + }, + "POST /api/v1/ask": { + ok: () => ({ path: `${V}/ask`, body: { question: "What is TypeScript?" } }), + bad: () => ({ path: `${V}/ask`, body: {} }), + }, + + "POST /api/v1/documents/:documentId/links": { + ok: (f) => ({ + path: doc(f, "/links"), + body: { targetDocumentId: f.docB, linkType: "references" }, + }), + bad: (f) => ({ path: doc(f, "/links"), body: { targetDocumentId: f.docB, linkType: "x" } }), + missing: (f) => ({ + path: `${V}/documents/nope/links`, + body: { targetDocumentId: f.docB, linkType: "related" }, + }), + }, + "DELETE /api/v1/links/:linkId": { + ok: (f) => ({ path: `${V}/links/${f.linkId}` }), + missing: () => ({ path: `${V}/links/nope` }), + }, + "GET /api/v1/links": { + ok: (f) => ({ path: `${V}/links`, query: { documentId: f.docA } }), + bad: () => ({ path: `${V}/links`, query: { linkType: "bogus" } }), + missing: () => ({ path: `${V}/links`, query: { documentId: "nope" } }), + }, + "GET /api/v1/documents/:documentId/prerequisites": { + ok: (f) => ({ path: doc(f, "/prerequisites") }), + missing: () => ({ path: `${V}/documents/nope/prerequisites` }), + }, + "GET /api/v1/graph": { + ok: () => ({ path: `${V}/graph`, query: { threshold: "0.5", topic: "Guides" } }), + bad: () => ({ path: `${V}/graph`, query: { threshold: "high" } }), + }, + + "POST /api/v1/documents/:documentId/tags": { + ok: (f) => ({ path: doc(f, "/tags"), body: { tags: ["new"] } }), + bad: (f) => ({ path: doc(f, "/tags"), body: { tags: [] } }), + missing: () => ({ path: `${V}/documents/nope/tags`, body: { tags: ["x"] } }), + }, + "DELETE /api/v1/documents/:documentId/tags": { + ok: (f) => ({ path: doc(f, "/tags"), query: { tags: "intro" } }), + bad: (f) => ({ path: doc(f, "/tags") }), + missing: () => ({ path: `${V}/documents/nope/tags`, query: { tags: "intro" } }), + }, + "GET /api/v1/tags": { ok: () => ({ path: `${V}/tags` }) }, + "GET /api/v1/documents/:documentId/suggested-tags": { + ok: (f) => ({ path: doc(f, "/suggested-tags") }), + bad: (f) => ({ path: doc(f, "/suggested-tags"), query: { limit: "0" } }), + missing: () => ({ path: `${V}/documents/nope/suggested-tags` }), + }, + + "GET /api/v1/topics": { + ok: () => ({ path: `${V}/topics` }), + missing: () => ({ path: `${V}/topics`, query: { parent: "nope" } }), + }, + "POST /api/v1/topics": { + ok: () => ({ path: `${V}/topics`, body: { name: "Recipes" } }), + bad: () => ({ path: `${V}/topics`, body: {} }), + }, + "DELETE /api/v1/topics/:topic": { + ok: () => ({ path: `${V}/topics/Guides` }), + bad: () => ({ path: `${V}/topics/Guides`, query: { deleteDocuments: "maybe" } }), + missing: () => ({ path: `${V}/topics/nope` }), + }, + + "POST /api/v1/searches": { + ok: () => ({ path: `${V}/searches`, body: { name: "ts", query: "typescript" } }), + bad: () => ({ path: `${V}/searches`, body: { name: "no query" } }), + }, + "GET /api/v1/searches": { + ok: () => ({ path: `${V}/searches` }), + bad: () => ({ path: `${V}/searches`, query: { offset: "-1" } }), + }, + "POST /api/v1/searches/:search/run": { + ok: () => ({ path: `${V}/searches/saved/run` }), + missing: () => ({ path: `${V}/searches/nope/run` }), + }, + "DELETE /api/v1/searches/:search": { + ok: () => ({ path: `${V}/searches/saved` }), + missing: () => ({ path: `${V}/searches/nope` }), + }, + + "POST /api/v1/packs": { + ok: () => ({ path: `${V}/packs`, body: { pack: "some-pack" } }), + okStatus: 202, + bad: () => ({ path: `${V}/packs`, body: { pack: "p", concurrency: 99 } }), + }, + "DELETE /api/v1/packs/:pack": { + ok: () => ({ path: `${V}/packs/demo-pack` }), + missing: () => ({ path: `${V}/packs/nope` }), + }, + "GET /api/v1/packs": { + ok: () => ({ path: `${V}/packs` }), + bad: () => ({ path: `${V}/packs`, query: { available: "maybe" } }), + missing: () => ({ path: `${V}/packs`, query: { available: "true", registry: "nope" } }), + }, + "GET /api/v1/registries": { + ok: () => ({ path: `${V}/registries` }), + }, + "GET /api/v1/registries/search": { + ok: () => ({ path: `${V}/registries/search`, query: { query: "react" } }), + bad: () => ({ path: `${V}/registries/search` }), + missing: () => ({ path: `${V}/registries/search`, query: { query: "x", registry: "nope" } }), + }, + + "GET /api/v1/connections": { ok: () => ({ path: `${V}/connections` }) }, + "POST /api/v1/sync": { + ok: () => ({ path: `${V}/sync`, body: { all: true } }), + okStatus: 202, + bad: () => ({ path: `${V}/sync`, body: { name: "bad name!" } }), + missing: () => ({ path: `${V}/sync`, body: { name: "nope" } }), + }, + "DELETE /api/v1/connections/:name": { + ok: () => ({ path: `${V}/connections/notes` }), + bad: () => ({ path: `${V}/connections/notes`, query: { type: "nope" } }), + missing: () => ({ path: `${V}/connections/nope` }), + }, + + "GET /api/v1/overview": { ok: () => ({ path: `${V}/overview` }) }, + "POST /api/v1/admin/reindex": { + ok: () => ({ path: `${V}/admin/reindex`, body: {} }), + okStatus: 202, + bad: () => ({ path: `${V}/admin/reindex`, body: { batchSize: 0 } }), + }, + "GET /api/v1/admin/duplicates": { + ok: () => ({ path: `${V}/admin/duplicates`, query: { strategy: "exact" } }), + okStatus: 202, + bad: () => ({ path: `${V}/admin/duplicates`, query: { threshold: "5" } }), + }, + "POST /api/v1/admin/prune-expired": { ok: () => ({ path: `${V}/admin/prune-expired` }) }, + "POST /api/v1/bulk/delete": { + ok: () => ({ path: `${V}/bulk/delete`, body: { topic: "Guides", dryRun: true } }), + bad: () => ({ path: `${V}/bulk/delete`, body: { topic: "Guides", dryRun: "yes" } }), + missing: () => ({ path: `${V}/bulk/delete`, body: { topic: "nope" } }), + }, + "POST /api/v1/bulk/retag": { + ok: () => ({ path: `${V}/bulk/retag`, body: { topic: "Guides", addTags: ["x"] } }), + bad: () => ({ path: `${V}/bulk/retag`, body: { topic: "Guides", addTags: "x" } }), + missing: () => ({ path: `${V}/bulk/retag`, body: { topic: "nope", addTags: ["x"] } }), + }, + "POST /api/v1/bulk/move": { + ok: () => ({ path: `${V}/bulk/move`, body: { library: "ts", targetTopic: "Guides" } }), + bad: () => ({ path: `${V}/bulk/move`, body: { library: "ts" } }), + missing: () => ({ path: `${V}/bulk/move`, body: { library: "ts", targetTopic: "nope" } }), + }, + + "GET /api/v1/analytics/popular": { + ok: () => ({ path: `${V}/analytics/popular` }), + bad: () => ({ path: `${V}/analytics/popular`, query: { limit: "many" } }), + }, + "GET /api/v1/analytics/stale": { + ok: () => ({ path: `${V}/analytics/stale`, query: { days: "7" } }), + bad: () => ({ path: `${V}/analytics/stale`, query: { days: "0" } }), + }, + "GET /api/v1/analytics/top-queries": { + ok: () => ({ path: `${V}/analytics/top-queries` }), + bad: () => ({ path: `${V}/analytics/top-queries`, query: { limit: "1000" } }), + }, + "GET /api/v1/analytics/searches": { + ok: () => ({ path: `${V}/analytics/searches` }), + bad: () => ({ path: `${V}/analytics/searches`, query: { days: "-3" } }), + }, + + "POST /api/v1/webhooks": { + ok: () => ({ + path: `${V}/webhooks`, + body: { url: "https://hooks.example.com/x", events: ["document.created"] }, + }), + bad: () => ({ path: `${V}/webhooks`, body: { url: "https://hooks.example.com/x" } }), + }, + "GET /api/v1/webhooks": { + ok: () => ({ path: `${V}/webhooks` }), + bad: () => ({ path: `${V}/webhooks`, query: { limit: "0" } }), + }, + "DELETE /api/v1/webhooks/:webhookId": { + ok: (f) => ({ path: `${V}/webhooks/${f.webhookId}` }), + missing: () => ({ path: `${V}/webhooks/nope` }), + }, + "POST /api/v1/webhooks/:webhookId/test": { + ok: (f) => ({ path: `${V}/webhooks/${f.webhookId}/test` }), + missing: () => ({ path: `${V}/webhooks/nope/test` }), + }, + + "GET /api/v1/tasks/:taskId": { + ok: (f) => ({ path: `${V}/tasks/${f.taskId}` }), + missing: () => ({ path: `${V}/tasks/nope` }), + }, + "POST /api/v1/tasks/:taskId/cancel": { + ok: (f) => ({ path: `${V}/tasks/${f.taskId}/cancel` }), + missing: () => ({ path: `${V}/tasks/nope/cancel` }), + }, + "GET /api/v1/tasks": { ok: () => ({ path: `${V}/tasks` }) }, +}; + +interface Reply { + status: number; + headers: Record<string, string | string[] | undefined>; + body: { data?: unknown; error?: { code: string; message: string }; [key: string]: unknown }; +} + +let server: Server; +let port: number; +let db: Database.Database; +let fixtures: Fixtures; + +/** Send a request with node:http (global fetch is stubbed by the webhook test). */ +function send(method: string, c: Case, headers: Record<string, string> = {}): Promise<Reply> { + const qs = c.query ? `?${new URLSearchParams(c.query).toString()}` : ""; + const payload = c.body === undefined ? undefined : JSON.stringify(c.body); + return new Promise((resolve, reject) => { + const req = request( + { + host: "127.0.0.1", + port, + method, + path: c.path + qs, + headers: { ...(payload ? { "Content-Type": "application/json" } : {}), ...headers }, + }, + (res) => { + let raw = ""; + res.setEncoding("utf8"); + res.on("data", (chunk: string) => (raw += chunk)); + res.on("end", () => + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: raw ? (JSON.parse(raw) as Reply["body"]) : {}, + }), + ); + }, + ); + req.on("error", reject); + req.end(payload); + }); +} + +const fakeLlm = { + model: "fake", + complete: (): Promise<{ text: string }> => Promise.resolve({ text: "An answer." }), +}; + +async function seed(): Promise<Fixtures> { + const provider = new MockEmbeddingProvider(); + const topic = createTopic(db, { name: "Guides" }); + const a = await indexDocument(db, provider, { + title: "TypeScript intro", + content: "TypeScript is a typed superset of JavaScript.", + sourceType: "manual", + topicId: topic.id, + }); + const b = await indexDocument(db, provider, { + title: "Node guide", + content: "Node.js runs JavaScript on the server.", + sourceType: "library", + library: "ts", + }); + db.prepare("INSERT INTO tags (id, name) VALUES ('t1', 'intro')").run(); + db.prepare("INSERT INTO document_tags (document_id, tag_id) VALUES (?, 't1')").run(a.id); + await updateDocument(db, provider, a.id, { content: "TypeScript adds static types." }); + const link = createLink(db, a.id, b.id, "related"); + createSavedSearch(db, "saved", "typescript"); + const hook = await createWebhook(db, "https://hooks.example.com/libscope", ["document.created"]); + db.prepare("INSERT INTO packs (name, version) VALUES ('demo-pack', '1.0.0')").run(); + saveConnectorSettings("obsidian", "notes", { + vaultPath: join(tempHome, "vault"), + topicMapping: "folder", + excludePatterns: [], + }); + const { task } = taskRegistry.create("operation", "test"); + return { docA: a.id, docB: b.id, linkId: link.id, webhookId: hook.id, taskId: task.id }; +} + +describe("REST routes generated from operations", () => { + let ctxRef: ReturnType<typeof createOperationContext>; + + beforeAll(async () => { + server = createServer((req, res) => void handleRequest(req, res, ctxRef)); + await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve)); + port = (server.address() as AddressInfo).port; + }); + + afterAll(async () => { + await new Promise<void>((resolve) => server.close(() => resolve())); + }); + + beforeEach(async () => { + tempHome = join(tmpdir(), `libscope-api-routes-${randomUUID()}`); + mkdirSync(join(tempHome, "vault"), { recursive: true }); + db = createTestDbWithVec(); + ctxRef = createOperationContext({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + surface: "api", + llm: fakeLlm, + }); + fixtures = await seed(); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response(null, { status: 204, statusText: "No Content" })), + ); + }); + + afterEach(async () => { + vi.unstubAllGlobals(); + // Let background tasks started by 202 routes settle before the database closes. + await new Promise((resolve) => setTimeout(resolve, 20)); + db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); + + it("has a contract case for every route, and every case names a route", () => { + const routes = API_ROUTES.map((r) => `${r.method} ${r.path}`).sort(); + expect(Object.keys(CASES).sort()).toEqual(routes); + }); + + for (const [key, cases] of Object.entries(CASES)) { + const [method = "GET"] = key.split(" "); + + it(`${key} succeeds`, async () => { + const reply = await send(method, cases.ok(fixtures)); + expect(reply.body.error, JSON.stringify(reply.body.error)).toBeUndefined(); + expect(reply.status).toBe(cases.okStatus ?? 200); + }); + + const { bad, missing } = cases; + if (bad) { + it(`${key} answers 400 for invalid input`, async () => { + const reply = await send(method, bad(fixtures)); + expect(reply.status, JSON.stringify(reply.body)).toBe(400); + expect(reply.body.error?.code).toMatch(/VALIDATION_ERROR|INVALID_JSON/); + }); + } + if (missing) { + it(`${key} answers 404 for a missing resource`, async () => { + const reply = await send(method, missing(fixtures)); + expect(reply.status, JSON.stringify(reply.body)).toBe(404); + expect(reply.body.error?.code).toMatch(/NOT_FOUND$/); + }); + } + } + + it("wraps results as { data, meta } with ids in the result", async () => { + const reply = await send("GET", { path: `${V}/documents/${fixtures.docA}` }); + expect(reply.body).toMatchObject({ + data: { document: { documentId: fixtures.docA, title: "TypeScript intro" } }, + meta: { took: expect.any(Number) as number }, + }); + }); + + it("runs a long-running operation as a task and reports its result", async () => { + const started = await send("POST", { + path: `${V}/documents`, + body: { title: "Queued", content: "Queued document body" }, + }); + expect(started.status).toBe(202); + const data = started.body.data as { taskId: string; operation: string }; + expect(data.operation).toBe("add"); + expect(started.headers["location"]).toBe(`${V}/tasks/${data.taskId}`); + + let task: { status: string; result?: string } = { status: "running" }; + await vi.waitFor(async () => { + const polled = await send("GET", { path: `${V}/tasks/${data.taskId}` }); + task = polled.body.data as typeof task; + expect(task.status).toBe("completed"); + }); + const result = JSON.parse(task.result ?? "null") as { documents: Array<{ title: string }> }; + expect(JSON.stringify(result)).toContain("Queued"); + }); + + it("coerces query values by the operation schema (numbers, booleans, arrays)", async () => { + const reply = await send("GET", { + path: `${V}/documents`, + query: { limit: "1", offset: "1" }, + }); + expect(reply.body.data).toMatchObject({ limit: 1, offset: 1, total: 2 }); + const tagged = await send("GET", { path: `${V}/documents`, query: { tags: "intro" } }); + expect((tagged.body.data as { total: number }).total).toBe(1); + }); + + it("refuses local paths over REST with 400 before any task starts", async () => { + const tasksBefore = taskRegistry.list().length; + for (const source of ["/etc/hosts", "/no/such/path"]) { + const reply = await send("POST", { path: `${V}/documents`, body: { source } }); + expect(reply.status).toBe(400); + expect(reply.body.error?.message).toContain("only available from the CLI"); + } + const noTopic = await send("POST", { + path: `${V}/documents`, + body: { title: "T", content: "C", topic: "nope" }, + }); + expect(noTopic.status).toBe(404); + expect(taskRegistry.list()).toHaveLength(tasksBefore); + }); + + it("answers 400 for a malformed JSON body and a non-object body", async () => { + const bad = await new Promise<Reply>((resolve, reject) => { + const req = request( + { host: "127.0.0.1", port, method: "POST", path: `${V}/topics` }, + (res) => { + let raw = ""; + res.on("data", (c: Buffer) => (raw += c.toString())); + res.on("end", () => + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: JSON.parse(raw) as Reply["body"], + }), + ); + }, + ); + req.on("error", reject); + req.end("{not json"); + }); + expect(bad.status).toBe(400); + expect(bad.body.error?.code).toBe("INVALID_JSON"); + const array = await send("POST", { path: `${V}/topics`, body: ["x"] }); + expect(array.status).toBe(400); + }); + + it("answers 404 for unknown routes and wrong methods", async () => { + expect((await send("GET", { path: `${V}/nope` })).status).toBe(404); + expect((await send("PUT", { path: `${V}/documents` })).status).toBe(404); + expect((await send("GET", { path: `${V}/documents/url/extra/more` })).status).toBe(404); + }); + + it("answers 400 for a malformed path escape", async () => { + const reply = await send("GET", { path: `${V}/documents/%E0%A4%A` }); + expect(reply.status).toBe(400); + }); + + it("answers 500 with the config hint when ask has no LLM", async () => { + ctxRef = createOperationContext({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + surface: "api", + llm: null, + }); + const reply = await send("POST", { path: `${V}/ask`, body: { question: "Why?" } }); + expect(reply.status).toBe(500); + expect(reply.body.error?.code).toBe("CONFIG_ERROR"); + }); +}); diff --git a/tests/unit/api-server.test.ts b/tests/unit/api-server.test.ts index 8ec7905b..e62579ff 100644 --- a/tests/unit/api-server.test.ts +++ b/tests/unit/api-server.test.ts @@ -3,6 +3,7 @@ import { createTestDbWithVec } from "../fixtures/test-db.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; import { initLogger } from "../../src/logger.js"; import type Database from "better-sqlite3"; +import { testConfig } from "./operations/helpers.js"; // Mock the scheduler to avoid real cron vi.mock("../../src/core/scheduler.js", () => ({ @@ -29,24 +30,30 @@ describe("startApiServer", () => { it("starts on port 0 and returns a close function", async () => { const provider = new MockEmbeddingProvider(); - const result = await startApiServer(db, provider, { - port: 0, - host: "127.0.0.1", - enableScheduler: false, - }); + const result = await startApiServer( + { db, provider, config: testConfig() }, + { + port: 0, + host: "127.0.0.1", + enableScheduler: false, + }, + ); - expect(result.port).toBe(0); + expect(result.port).toBeGreaterThan(0); expect(typeof result.close).toBe("function"); await result.close(); }); it("starts with scheduler enabled but no entries", async () => { const provider = new MockEmbeddingProvider(); - const result = await startApiServer(db, provider, { - port: 0, - host: "127.0.0.1", - enableScheduler: true, - }); + const result = await startApiServer( + { db, provider, config: testConfig() }, + { + port: 0, + host: "127.0.0.1", + enableScheduler: true, + }, + ); expect(typeof result.close).toBe("function"); expect(result.scheduler).toBeUndefined(); @@ -55,12 +62,45 @@ describe("startApiServer", () => { it("starts with default options (scheduler enabled by default)", async () => { const provider = new MockEmbeddingProvider(); - const result = await startApiServer(db, provider, { - port: 0, - host: "127.0.0.1", - }); + const result = await startApiServer( + { db, provider, config: testConfig() }, + { + port: 0, + host: "127.0.0.1", + }, + ); expect(typeof result.close).toBe("function"); await result.close(); }); + + it("serves generated routes behind CORS, origin and API key checks", async () => { + const provider = new MockEmbeddingProvider(); + const server = await startApiServer( + { db, provider, config: testConfig() }, + { port: 0, host: "127.0.0.1", enableScheduler: false, corsOrigins: ["http://app.example"] }, + ); + const base = `http://127.0.0.1:${server.port}/api/v1`; + try { + const health = await fetch(`${base}/health`, { headers: { Origin: "http://app.example" } }); + expect(health.status).toBe(200); + expect(health.headers.get("access-control-allow-origin")).toBe("http://app.example"); + expect(health.headers.get("content-security-policy")).toContain("default-src 'self'"); + + const foreign = await fetch(`${base}/topics`, { + method: "POST", + headers: { Origin: "http://evil.example", "Content-Type": "text/plain" }, + body: JSON.stringify({ name: "x" }), + }); + expect(foreign.status).toBe(403); + + process.env["LIBSCOPE_API_KEY"] = "k"; + expect((await fetch(`${base}/overview`)).status).toBe(401); + const authed = await fetch(`${base}/overview`, { headers: { Authorization: "Bearer k" } }); + expect(authed.status).toBe(200); + } finally { + delete process.env["LIBSCOPE_API_KEY"]; + await server.close(); + } + }); }); diff --git a/tests/unit/api.test.ts b/tests/unit/api.test.ts index a9a42859..6386c097 100644 --- a/tests/unit/api.test.ts +++ b/tests/unit/api.test.ts @@ -1,12 +1,9 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { describe, it, expect, afterEach } from "vitest"; import { IncomingMessage, ServerResponse } from "node:http"; import { Socket } from "node:net"; -import type Database from "better-sqlite3"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; -import { createTestDbWithVec } from "../fixtures/test-db.js"; -import { handleRequest } from "../../src/api/routes.js"; import { corsMiddleware, + rejectForeignWrite, parseJsonBody, sendJson, sendError, @@ -14,18 +11,10 @@ import { checkApiKey, getRateLimitMapSize, MAX_RATE_LIMIT_ENTRIES, + setSecurityHeaders, } from "../../src/api/middleware.js"; -import { OPENAPI_SPEC } from "../../src/api/openapi.js"; -import { indexDocument } from "../../src/core/indexing.js"; -import { createTopic } from "../../src/core/topics.js"; - -vi.mock("../../src/core/scheduler.js", async (importOriginal) => { - const orig = await importOriginal<typeof import("../../src/core/scheduler.js")>(); - return { - ...orig, - loadScheduleEntries: vi.fn(() => []), - }; -}); +import { API_ROUTES, OPENAPI_PATH } from "../../src/api/routes.js"; +import { buildOpenApiSpec, toOpenApiPath } from "../../src/api/openapi.js"; interface ApiResponse { data?: Record<string, unknown>; @@ -124,38 +113,62 @@ function createMockRes(): MockRes { describe("API middleware", () => { describe("corsMiddleware", () => { - it("should handle OPTIONS preflight", () => { - const req = createMockReq("OPTIONS", "/api/v1/health"); + it("answers OPTIONS preflight for a listed origin with every method", () => { + const req = createMockReq("OPTIONS", "/api/v1/documents"); + req.headers["origin"] = "http://localhost:3000"; const { res, getStatus, getHeaders } = createMockRes(); - const handled = corsMiddleware(req, res, ["*"]); + const handled = corsMiddleware(req, res, ["http://localhost:3000"]); expect(handled).toBe(true); expect(getStatus()).toBe(204); - expect(getHeaders()["Access-Control-Allow-Origin"]).toBe("*"); + expect(getHeaders()["Access-Control-Allow-Origin"]).toBe("http://localhost:3000"); expect(getHeaders()["Access-Control-Allow-Methods"]).toBe( - "GET, POST, PATCH, DELETE, OPTIONS", + "GET, POST, PATCH, PUT, DELETE, OPTIONS", ); + expect(getHeaders()["Vary"]).toBe("Origin"); }); - it("should set CORS headers for non-preflight requests", () => { - const req = createMockReq("GET", "/api/v1/health"); + it("lets a wildcard origin read but not write", () => { + const req = createMockReq("OPTIONS", "/api/v1/documents"); + req.headers["origin"] = "http://evil.example"; const { res, getHeaders } = createMockRes(); - const handled = corsMiddleware(req, res, ["*"]); + corsMiddleware(req, res, ["*"]); - expect(handled).toBe(false); expect(getHeaders()["Access-Control-Allow-Origin"]).toBe("*"); + expect(getHeaders()["Access-Control-Allow-Methods"]).toBe("GET, OPTIONS"); }); - it("should restrict origin when not wildcard", () => { + it("sets no allow-origin for an unlisted origin", () => { const req = createMockReq("GET", "/api/v1/health"); - req.headers["origin"] = "http://example.com"; + req.headers["origin"] = "http://evil.example"; const { res, getHeaders } = createMockRes(); - corsMiddleware(req, res, ["http://example.com"]); + expect(corsMiddleware(req, res, ["http://localhost"])).toBe(false); + expect(getHeaders()["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + }); + + describe("rejectForeignWrite", () => { + function check(method: string, origin: string | undefined, origins: string[]): number | null { + const req = createMockReq(method, "/api/v1/documents/x"); + if (origin !== undefined) req.headers["origin"] = origin; + const { res, getStatus } = createMockRes(); + return rejectForeignWrite(req, res, origins) ? getStatus() : null; + } + + it('rejects writes from unlisted origins, also when "*" is allowed', () => { + expect(check("DELETE", "http://evil.example", [])).toBe(403); + expect(check("POST", "http://evil.example", ["*"])).toBe(403); + }); - expect(getHeaders()["Access-Control-Allow-Origin"]).toBe("http://example.com"); + it("allows reads, same-origin writes, listed origins and non-browser clients", () => { + expect(check("GET", "http://evil.example", [])).toBeNull(); + expect(check("DELETE", "http://localhost:3378", [])).toBeNull(); + expect(check("POST", "http://app.example", ["http://app.example"])).toBeNull(); + expect(check("PATCH", undefined, [])).toBeNull(); + expect(check("POST", "null", [])).toBe(403); }); }); @@ -168,7 +181,7 @@ describe("API middleware", () => { it("should reject invalid JSON", async () => { const req = createMockReq("POST", "/test", "not json{{{"); - await expect(parseJsonBody(req)).rejects.toThrow("Invalid JSON body"); + await expect(parseJsonBody(req)).rejects.toThrow("Request body contains invalid JSON"); }); it("should return null for empty body", async () => { @@ -199,627 +212,56 @@ describe("API middleware", () => { }); }); -describe("API routes", () => { - let db: Database.Database; - let provider: MockEmbeddingProvider; - - beforeEach(() => { - db = createTestDbWithVec(); - provider = new MockEmbeddingProvider(); - }); - - describe("GET /api/v1/health", () => { - it("should return health status", async () => { - const req = createMockReq("GET", "/api/v1/health"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data.status).toBe("ok"); - expect(typeof parsed.data.docCount).toBe("number"); - expect(typeof parsed.meta.took).toBe("number"); - }); - }); - - describe("GET /openapi.json", () => { - it("should return the OpenAPI spec", async () => { - const req = createMockReq("GET", "/openapi.json"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.openapi).toBe("3.0.3"); - expect(parsed.info.title).toBe("LibScope REST API"); - }); - }); - - describe("GET /api/v1/search", () => { - it("should return 400 without query param", async () => { - const req = createMockReq("GET", "/api/v1/search"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("VALIDATION_ERROR"); - }); - - it("should search with query params", async () => { - await indexDocument(db, provider, { - title: "Test Doc", - content: "Hello world content", - sourceType: "manual", - }); - - const req = createMockReq("GET", "/api/v1/search?q=hello&limit=5"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data).toBeDefined(); - expect(parsed.meta.took).toBeDefined(); - }); - - it("should accept offset query parameter for pagination", async () => { - await indexDocument(db, provider, { - title: "Doc A", - content: "First document content", - sourceType: "manual", - }); - await indexDocument(db, provider, { - title: "Doc B", - content: "Second document content", - sourceType: "manual", - }); - - const req = createMockReq("GET", "/api/v1/search?q=document&limit=5&offset=1"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data).toBeDefined(); - expect(parsed.meta.took).toBeDefined(); - }); - }); - - describe("POST /api/v1/documents", () => { - it("should index a new document", async () => { - const req = createMockReq("POST", "/api/v1/documents", { - title: "My Doc", - content: "Some content here", - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(201); - const parsed = parseResponse(getBody()); - expect(parsed.data.id).toBeDefined(); - expect(parsed.data.chunkCount).toBeGreaterThanOrEqual(1); - }); - - it("should return 400 for missing fields", async () => { - const req = createMockReq("POST", "/api/v1/documents", { title: "No content" }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("VALIDATION_ERROR"); - }); - - it("should index document with tags", async () => { - const req = createMockReq("POST", "/api/v1/documents", { - title: "Tagged Doc", - content: "Content with tags", - tags: ["typescript", "api"], - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(201); - const parsed = parseResponse(getBody()); - expect(parsed.data.id).toBeDefined(); - }); - - it("should return 400 for invalid JSON body", async () => { - const req = createMockReq("POST", "/api/v1/documents", "not-valid-json{{{"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("INVALID_JSON"); - }); - }); - - describe("GET /api/v1/documents/:id", () => { - it("should return a document by ID", async () => { - const doc = await indexDocument(db, provider, { - title: "Fetch Me", - content: "Document content", - sourceType: "manual", - }); - - const req = createMockReq("GET", `/api/v1/documents/${doc.id}`); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data.id).toBe(doc.id); - expect(parsed.data.title).toBe("Fetch Me"); - }); - - it("should return 404 for non-existent document", async () => { - const req = createMockReq("GET", "/api/v1/documents/nonexistent-id"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(404); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("NOT_FOUND"); - }); - }); - - describe("DELETE /api/v1/documents/:id", () => { - it("should delete a document", async () => { - const doc = await indexDocument(db, provider, { - title: "Delete Me", - content: "To be deleted", - sourceType: "manual", - }); - - const req = createMockReq("DELETE", `/api/v1/documents/${doc.id}`); - const { res, getStatus } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(204); - }); - - it("should return 404 for deleting non-existent document", async () => { - const req = createMockReq("DELETE", "/api/v1/documents/nonexistent-id"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(404); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("NOT_FOUND"); - }); - }); - - describe("POST /api/v1/ask", () => { - it("should return 400 without question", async () => { - const req = createMockReq("POST", "/api/v1/ask", { notQuestion: "test" }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("VALIDATION_ERROR"); - }); - - it("should return SSE stream when Accept: text/event-stream", async () => { - // Mock the LLM and search modules to avoid real calls - const ragModule = await import("../../src/core/rag.js"); - const searchModule = await import("../../src/core/search.js"); - const configModule = await import("../../src/config.js"); - - vi.spyOn(configModule, "loadConfig").mockReturnValue({ - embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, - database: { path: ":memory:" }, - indexing: { maxDocumentSize: 1024 }, - logging: { level: "silent" }, - }); - - vi.spyOn(searchModule, "searchDocuments").mockResolvedValue({ - results: [], - totalCount: 0, - }); - - vi.spyOn(ragModule, "createLlmProvider").mockReturnValue({ - model: "mock-model", - complete: vi.fn().mockResolvedValue({ text: "SSE answer", tokensUsed: 10 }), - }); - - const req = createMockReq("POST", "/api/v1/ask", { question: "What is SSE?" }); - req.headers["accept"] = "text/event-stream"; - const { res, getStatus, getBody, getHeaders } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - expect(getHeaders()["Content-Type"]).toBe("text/event-stream"); - expect(getHeaders()["Cache-Control"]).toBe("no-cache"); - expect(getHeaders()["Connection"]).toBe("keep-alive"); - - const body = getBody(); - // Should contain at least one token event and a done event - expect(body).toContain("data: "); - expect(body).toContain('"token"'); - expect(body).toContain('"done":true'); - expect(body).toContain('"sources"'); - - // Parse individual SSE events - const events = body - .split("\n\n") - .filter((line: string) => line.startsWith("data: ")) - .map((line: string) => JSON.parse(line.replace("data: ", "")) as Record<string, unknown>); - - expect(events.length).toBeGreaterThanOrEqual(2); - expect(events[0]).toHaveProperty("token"); - expect(events[events.length - 1]).toMatchObject({ done: true }); - - vi.mocked(configModule.loadConfig).mockRestore(); - vi.mocked(searchModule.searchDocuments).mockRestore(); - vi.mocked(ragModule.createLlmProvider).mockRestore(); - }); - - it("should return normal JSON when Accept header is not SSE", async () => { - const req = createMockReq("POST", "/api/v1/ask", { question: "test" }); - req.headers["accept"] = "application/json"; - const { res, getHeaders } = createMockRes(); - - // This will fail because no LLM is configured, but it should NOT return SSE headers - await handleRequest(req, res, db, provider); - - expect(getHeaders()["Content-Type"]).not.toBe("text/event-stream"); - }); - }); - - describe("GET /api/v1/documents", () => { - it("should list documents", async () => { - await indexDocument(db, provider, { - title: "Doc 1", - content: "Content 1", - sourceType: "manual", - }); - - const req = createMockReq("GET", "/api/v1/documents"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - expect(parsed.data.length).toBeGreaterThanOrEqual(1); - }); - }); - - describe("GET /api/v1/topics", () => { - it("should list topics", async () => { - createTopic(db, { name: "test-topic" }); - - const req = createMockReq("GET", "/api/v1/topics"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - }); - }); - - describe("POST /api/v1/topics", () => { - it("should create a topic", async () => { - const req = createMockReq("POST", "/api/v1/topics", { name: "new-topic" }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(201); - const parsed = parseResponse(getBody()); - expect(parsed.data.name).toBe("new-topic"); - }); - - it("should return 400 without name", async () => { - const req = createMockReq("POST", "/api/v1/topics", { description: "no name" }); - const { res, getStatus } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - }); - }); - - describe("GET /api/v1/tags", () => { - it("should list tags", async () => { - const req = createMockReq("GET", "/api/v1/tags"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - }); - }); - - describe("POST /api/v1/documents/:id/tags", () => { - it("should add tags to a document", async () => { - const doc = await indexDocument(db, provider, { - title: "Tag Target", - content: "Content", - sourceType: "manual", - }); - - const req = createMockReq("POST", `/api/v1/documents/${doc.id}/tags`, { - tags: ["alpha", "beta"], - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - }); - }); - - describe("GET /api/v1/stats", () => { - it("should return stats", async () => { - const req = createMockReq("GET", "/api/v1/stats"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(typeof parsed.data.totalDocuments).toBe("number"); - }); - - it("should return databaseSizeBytes field", async () => { - const req = createMockReq("GET", "/api/v1/stats"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(typeof parsed.data.databaseSizeBytes).toBe("number"); - }); - }); - - describe("Connector status endpoint", () => { - it("GET /api/v1/connectors/status should return empty array when no syncs", async () => { - const req = createMockReq("GET", "/api/v1/connectors/status"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(parsed.data).toEqual([]); - }); - - it("GET /api/v1/connectors/status should return latest sync status", async () => { - // Insert a sync record directly - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, completed_at, status, docs_added) - VALUES ('obsidian', 'vault1', datetime('now'), datetime('now'), 'completed', 5)`, - ).run(); - - const req = createMockReq("GET", "/api/v1/connectors/status"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const data = parsed.data as unknown as Array<Record<string, unknown>>; - expect(data).toHaveLength(1); - expect(data[0].connector_type).toBe("obsidian"); - expect(data[0].docs_added).toBe(5); - }); - - it("GET /api/v1/connectors/status?history=true should return full history", async () => { - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, status) - VALUES ('obsidian', 'vault1', datetime('now'), 'running')`, - ).run(); - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, completed_at, status) - VALUES ('obsidian', 'vault1', datetime('now'), datetime('now'), 'completed')`, - ).run(); - - const req = createMockReq("GET", "/api/v1/connectors/status?history=true"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const data = parsed.data as unknown as Array<Record<string, unknown>>; - expect(data).toHaveLength(2); - }); - - it("GET /api/v1/connectors/status?type=notion should filter by type", async () => { - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, status) - VALUES ('obsidian', 'vault1', datetime('now'), 'completed')`, - ).run(); - db.prepare( - `INSERT INTO connector_syncs (connector_type, connector_name, started_at, status) - VALUES ('notion', 'notion', datetime('now'), 'completed')`, - ).run(); - - const req = createMockReq("GET", "/api/v1/connectors/status?type=notion"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const data = parsed.data as unknown as Array<Record<string, unknown>>; - expect(data).toHaveLength(1); - expect(data[0].connector_type).toBe("notion"); - }); - }); - - describe("Connector schedules", () => { - it("GET /api/v1/connectors/schedules should return schedule entries", async () => { - const { loadScheduleEntries } = await import("../../src/core/scheduler.js"); - const mockLoad = vi.mocked(loadScheduleEntries); - mockLoad.mockReturnValueOnce([ - { connectorType: "notion", connectorName: "notion", cronExpression: "0 */6 * * *" }, - ]); - - const req = createMockReq("GET", "/api/v1/connectors/schedules"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const schedules = (parsed.data as unknown as { schedules: unknown[] }).schedules; - expect(schedules).toHaveLength(1); - }); - - it("GET /api/v1/connectors/schedules should return empty array when none configured", async () => { - const { loadScheduleEntries } = await import("../../src/core/scheduler.js"); - const mockLoad = vi.mocked(loadScheduleEntries); - mockLoad.mockReturnValueOnce([]); - - const req = createMockReq("GET", "/api/v1/connectors/schedules"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - const schedules = (parsed.data as unknown as { schedules: unknown[] }).schedules; - expect(schedules).toHaveLength(0); - }); - }); - - describe("Unknown route", () => { - it("should return 404", async () => { - const req = createMockReq("GET", "/api/v1/nonexistent"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(404); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("NOT_FOUND"); - }); - }); - - describe("Webhooks API", () => { - it("should create a webhook via POST /api/v1/webhooks", async () => { - process.env.LIBSCOPE_SECRET_KEY = "test-key"; - try { - const req = createMockReq("POST", "/api/v1/webhooks", { - url: "https://example.com/hook", - events: ["document.created"], - secret: "my-secret", - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(201); - const parsed = parseResponse(getBody()); - expect(parsed.data.url).toBe("https://example.com/hook"); - expect(parsed.data.hasSecret).toBe(true); - expect(parsed.data.secret).toBeUndefined(); - } finally { - delete process.env.LIBSCOPE_SECRET_KEY; - } - }); - - it("should list webhooks via GET /api/v1/webhooks", async () => { - // Create one first - const createReq = createMockReq("POST", "/api/v1/webhooks", { - url: "https://example.com/hook", - events: ["document.created"], - }); - const { res: createRes } = createMockRes(); - await handleRequest(createReq, createRes, db, provider); - - const req = createMockReq("GET", "/api/v1/webhooks"); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(200); - const parsed = parseResponse(getBody()); - expect(Array.isArray(parsed.data)).toBe(true); - expect(parsed.data.length).toBeGreaterThanOrEqual(1); - }); - - it("should return 400 when creating webhook without url", async () => { - const req = createMockReq("POST", "/api/v1/webhooks", { - events: ["document.created"], - }); - const { res, getStatus, getBody } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(400); - const parsed = parseResponse(getBody()); - expect(parsed.error.code).toBe("VALIDATION_ERROR"); - }); - - it("should delete a webhook via DELETE /api/v1/webhooks/:id", async () => { - const createReq = createMockReq("POST", "/api/v1/webhooks", { - url: "https://example.com/hook", - events: ["document.created"], - }); - const { res: createRes, getBody: getCreateBody } = createMockRes(); - await handleRequest(createReq, createRes, db, provider); - const created = parseResponse(getCreateBody()); - const id = (created.data as Record<string, unknown>).id as string; - - const req = createMockReq("DELETE", `/api/v1/webhooks/${id}`); - const { res, getStatus } = createMockRes(); - - await handleRequest(req, res, db, provider); - - expect(getStatus()).toBe(204); - }); - }); -}); - describe("OpenAPI spec", () => { - it("should have valid structure", () => { - expect(OPENAPI_SPEC.openapi).toBe("3.0.3"); - expect(OPENAPI_SPEC.info.title).toBeDefined(); - expect(OPENAPI_SPEC.paths).toBeDefined(); - expect(OPENAPI_SPEC.components).toBeDefined(); - }); - - it("should define all endpoints", () => { - const paths = Object.keys(OPENAPI_SPEC.paths); - expect(paths).toContain("/api/v1/search"); - expect(paths).toContain("/api/v1/documents"); - expect(paths).toContain("/api/v1/documents/{id}"); - expect(paths).toContain("/api/v1/ask"); - expect(paths).toContain("/api/v1/topics"); - expect(paths).toContain("/api/v1/tags"); - expect(paths).toContain("/api/v1/stats"); - expect(paths).toContain("/api/v1/health"); - expect(paths).toContain("/openapi.json"); + const spec = buildOpenApiSpec(API_ROUTES) as { + openapi: string; + info: { title: string; version: string }; + paths: Record<string, Record<string, Record<string, unknown>>>; + }; + const specRoutes = Object.entries(spec.paths).flatMap(([path, methods]) => + Object.keys(methods).map((method) => `${method.toUpperCase()} ${path}`), + ); + + it("is OpenAPI 3.1 with the package version", () => { + expect(spec.openapi).toBe("3.1.0"); + expect(spec.info.version).toMatch(/^\d+\.\d+\.\d+/); + }); + + it("documents every route in the router, and nothing else", () => { + const routerRoutes = API_ROUTES.map((r) => `${r.method} ${toOpenApiPath(r.path)}`); + expect([...specRoutes].sort()).toEqual([...routerRoutes].sort()); + expect(specRoutes).toContain(`GET ${OPENAPI_PATH}`); + }); + + it("takes parameters and summaries from the operation schemas", () => { + const search = spec.paths["/api/v1/search"]?.["get"] as { + operationId: string; + summary: string; + parameters: Array<{ name: string; in: string; schema: { type?: string } }>; + }; + expect(search.operationId).toBe("search"); + const limit = search.parameters.find((p) => p.name === "limit"); + expect(limit).toMatchObject({ in: "query", schema: { type: "integer", default: 10 } }); + const getDoc = spec.paths["/api/v1/documents/{documentId}"]?.["get"] as { + parameters: Array<{ name: string; in: string; required: boolean }>; + }; + expect(getDoc.parameters).toContainEqual( + expect.objectContaining({ name: "documentId", in: "path", required: true }), + ); + const add = spec.paths["/api/v1/documents"]?.["post"] as { + requestBody: { content: Record<string, { schema: { properties: object } }> }; + responses: Record<string, unknown>; + }; + expect(add.requestBody.content["application/json"]?.schema.properties).toHaveProperty("url"); + expect(add.responses).toHaveProperty("202"); + const link = spec.paths["/api/v1/documents/{documentId}/links"]?.["post"] as { + requestBody: { + content: Record<string, { schema: { properties: object; required: string[] } }>; + }; + }; + const body = link.requestBody.content["application/json"]?.schema; + expect(body?.properties).not.toHaveProperty("documentId"); + expect(body?.required).toEqual(["targetDocumentId", "linkType"]); }); }); @@ -843,17 +285,20 @@ describe("middleware — security", () => { expect(result).toEqual(data); }); - it("should set security headers via CORS middleware", () => { + it("should set security headers", () => { const socket = new Socket(); const req = new IncomingMessage(socket); req.method = "GET"; const res = new ServerResponse(req); - corsMiddleware(req, res, ["*"]); + setSecurityHeaders(res); expect(res.getHeader("X-Content-Type-Options")).toBe("nosniff"); expect(res.getHeader("X-Frame-Options")).toBe("DENY"); expect(res.getHeader("X-XSS-Protection")).toBe("1; mode=block"); expect(res.getHeader("Referrer-Policy")).toBe("strict-origin-when-cross-origin"); expect(res.getHeader("Content-Security-Policy")).toBeDefined(); + const noCsp = new ServerResponse(req); + setSecurityHeaders(noCsp, false); + expect(noCsp.getHeader("Content-Security-Policy")).toBeUndefined(); }); }); @@ -955,197 +400,3 @@ describe("middleware — rate limiting", () => { // Entry exists; size is at least 1 }); }); - -describe("Saved Searches API", () => { - let db: Database.Database; - const provider = new MockEmbeddingProvider(); - - beforeEach(async () => { - db = createTestDbWithVec(); - await indexDocument(db, provider, { - title: "Test Doc", - content: "# Test\n\nSome test content for searching.", - sourceType: "manual", - }); - }); - - afterEach(() => db.close()); - - it("POST /api/v1/searches creates a saved search", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/searches", { name: "my search", query: "test" }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(201); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); - - it("POST /api/v1/searches returns 400 without name", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/searches", { query: "test" }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(400); - const body = parseResponse(getBody()); - expect(body.error?.code).toBe("VALIDATION_ERROR"); - }); - - it("GET /api/v1/searches lists saved searches", async () => { - // First create one - const req1 = createMockReq("POST", "/api/v1/searches", { name: "my search", query: "test" }); - const mock1 = createMockRes(); - await handleRequest(req1, mock1.res, db, provider); - - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("GET", "/api/v1/searches"); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); - - it("DELETE /api/v1/searches/:id deletes a saved search", async () => { - // Create one first - const req1 = createMockReq("POST", "/api/v1/searches", { name: "del search", query: "test" }); - const mock1 = createMockRes(); - await handleRequest(req1, mock1.res, db, provider); - const created = parseResponse(mock1.getBody()); - const id = (created.data as Record<string, unknown>)?.id as string; - - const { res, getStatus } = createMockRes(); - const req = createMockReq("DELETE", `/api/v1/searches/${id}`); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(204); - }); - - it("POST /api/v1/searches/:id/run runs a saved search", async () => { - // Create one first - const req1 = createMockReq("POST", "/api/v1/searches", { name: "run search", query: "test" }); - const mock1 = createMockRes(); - await handleRequest(req1, mock1.res, db, provider); - const created = parseResponse(mock1.getBody()); - const id = (created.data as Record<string, unknown>)?.id as string; - - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", `/api/v1/searches/${id}/run`); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); -}); - -describe("Bulk Operations API", () => { - let db: Database.Database; - const provider = new MockEmbeddingProvider(); - - beforeEach(async () => { - db = createTestDbWithVec(); - await indexDocument(db, provider, { - title: "Bulk Doc", - content: "# Bulk\n\nContent for bulk ops.", - sourceType: "manual", - }); - }); - - afterEach(() => db.close()); - - it("POST /api/v1/bulk/delete returns 400 without selector", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/delete", {}); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(400); - const body = parseResponse(getBody()); - expect(body.error?.code).toBe("VALIDATION_ERROR"); - }); - - it("POST /api/v1/bulk/delete dry run", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/delete", { - selector: { sourceType: "manual" }, - dryRun: true, - }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); - - it("POST /api/v1/bulk/retag adds tags", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/retag", { - selector: { sourceType: "manual" }, - addTags: ["new-tag"], - dryRun: true, - }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); - - it("POST /api/v1/bulk/move requires targetTopicId", async () => { - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/move", { - selector: { sourceType: "manual" }, - }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(400); - const body = parseResponse(getBody()); - expect(body.error?.code).toBe("VALIDATION_ERROR"); - }); - - it("POST /api/v1/bulk/move with targetTopicId", async () => { - const topic = createTopic(db, { name: "Bulk Target" }); - const { res, getStatus, getBody } = createMockRes(); - const req = createMockReq("POST", "/api/v1/bulk/move", { - selector: { sourceType: "manual" }, - targetTopicId: topic.id, - dryRun: true, - }); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(200); - const body = parseResponse(getBody()); - expect(body.data).toBeDefined(); - }); -}); - -describe("Links API", () => { - let db: Database.Database; - const provider = new MockEmbeddingProvider(); - - beforeEach(() => { - db = createTestDbWithVec(); - }); - - afterEach(() => db.close()); - - it("DELETE /api/v1/links/:id deletes a link", async () => { - // Create two docs and link them - const doc1 = await indexDocument(db, provider, { - title: "Doc A", - content: "# A\n\nContent A.", - sourceType: "manual", - }); - const doc2 = await indexDocument(db, provider, { - title: "Doc B", - content: "# B\n\nContent B.", - sourceType: "manual", - }); - - // Create a link via the API: POST /api/v1/documents/:id/links - const { res: createRes, getBody: createBody } = createMockRes(); - const createReq = createMockReq("POST", `/api/v1/documents/${doc1.id}/links`, { - targetId: doc2.id, - linkType: "related", - }); - await handleRequest(createReq, createRes, db, provider); - const created = parseResponse(createBody()); - const linkId = (created.data as Record<string, unknown>)?.id as string; - - // Delete the link - const { res, getStatus } = createMockRes(); - const req = createMockReq("DELETE", `/api/v1/links/${linkId}`); - await handleRequest(req, res, db, provider); - expect(getStatus()).toBe(204); - }); -}); diff --git a/tests/unit/async-utils.test.ts b/tests/unit/async-utils.test.ts new file mode 100644 index 00000000..1b6021f4 --- /dev/null +++ b/tests/unit/async-utils.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { forEachSequential, mapSequential, runConcurrent } from "../../src/utils/async.js"; + +const tick = (): Promise<void> => new Promise((resolve) => setTimeout(resolve, 1)); + +describe("forEachSequential", () => { + it("runs one call at a time, in order, with the index", async () => { + const events: string[] = []; + let running = 0; + await forEachSequential(["a", "b", "c"], async (item, index) => { + running++; + expect(running).toBe(1); + events.push(`start ${item}${index}`); + await tick(); + events.push(`end ${item}${index}`); + running--; + }); + expect(events).toEqual(["start a0", "end a0", "start b1", "end b1", "start c2", "end c2"]); + }); + + it("stops at the first rejection", async () => { + const seen: number[] = []; + const run = forEachSequential([1, 2, 3], async (n) => { + seen.push(n); + await tick(); + if (n === 2) throw new Error("stop at 2"); + }); + await expect(run).rejects.toThrow("stop at 2"); + expect(seen).toEqual([1, 2]); + }); + + it("turns a synchronous throw into a rejection", async () => { + const run = forEachSequential([1], () => { + throw new Error("sync"); + }); + await expect(run).rejects.toThrow("sync"); + }); + + it("accepts any iterable and resolves at once when it is empty", async () => { + const seen: string[] = []; + await forEachSequential(new Set(["x", "y"]), async (item) => { + seen.push(item); + await tick(); + }); + await forEachSequential([], () => Promise.reject(new Error("not called"))); + expect(seen).toEqual(["x", "y"]); + }); +}); + +describe("mapSequential", () => { + it("resolves to the results in input order", async () => { + const delays = [5, 1, 3]; + const results = await mapSequential(delays, async (ms, index) => { + await new Promise((resolve) => setTimeout(resolve, ms)); + return `${index}:${ms}`; + }); + expect(results).toEqual(["0:5", "1:1", "2:3"]); + }); + + it("returns an empty array for no items", async () => { + await expect(mapSequential([], () => Promise.resolve(1))).resolves.toEqual([]); + }); +}); + +describe("runConcurrent", () => { + it("keeps at most `concurrency` tasks in flight and preserves order", async () => { + let running = 0; + let maxRunning = 0; + const tasks = [4, 1, 3, 2, 1].map((ms, i) => async (): Promise<number> => { + running++; + maxRunning = Math.max(maxRunning, running); + await new Promise((resolve) => setTimeout(resolve, ms)); + running--; + return i; + }); + await expect(runConcurrent(tasks, 2)).resolves.toEqual([0, 1, 2, 3, 4]); + expect(maxRunning).toBe(2); + }); + + it("handles more workers than tasks and an empty task list", async () => { + const tasks = [(): Promise<string> => Promise.resolve("only")]; + await expect(runConcurrent(tasks, 5)).resolves.toEqual(["only"]); + await expect(runConcurrent([], 3)).resolves.toEqual([]); + }); + + it("rejects when a task rejects", async () => { + const tasks: Array<() => Promise<number>> = [ + (): Promise<number> => Promise.resolve(1), + (): Promise<number> => Promise.reject(new Error("task failed")), + ]; + await expect(runConcurrent(tasks, 2)).rejects.toThrow("task failed"); + }); +}); diff --git a/tests/unit/batch-search.test.ts b/tests/unit/batch-search.test.ts deleted file mode 100644 index b887d809..00000000 --- a/tests/unit/batch-search.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import type Database from "better-sqlite3"; -import { createTestDbWithVec } from "../fixtures/test-db.js"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; -import { seedTestDocument, insertChunk } from "../fixtures/helpers.js"; -import { - searchBatch, - BATCH_SEARCH_MAX_REQUESTS, - type BatchSearchRequest, -} from "../../src/core/batch-search.js"; - -describe("searchBatch", () => { - let db: Database.Database; - let provider: MockEmbeddingProvider; - - beforeEach(() => { - db = createTestDbWithVec(); - provider = new MockEmbeddingProvider(); - - // Seed a document with a chunk so searches have something to find - seedTestDocument(db, "doc-1", { title: "TypeScript Guide", content: "TypeScript basics" }); - insertChunk(db, "chunk-1", "doc-1", "TypeScript is a typed superset of JavaScript", 0); - }); - - afterEach(() => { - db.close(); - }); - - it("should throw ValidationError for empty request array", async () => { - await expect(searchBatch(db, provider, [])).rejects.toThrow( - "At least one search request is required", - ); - }); - - it("should throw ValidationError for non-array input", async () => { - await expect( - searchBatch(db, provider, null as unknown as BatchSearchRequest[]), - ).rejects.toThrow("At least one search request is required"); - }); - - it("should throw ValidationError when exceeding max requests", async () => { - const requests: BatchSearchRequest[] = Array.from( - { length: BATCH_SEARCH_MAX_REQUESTS + 1 }, - (_, i) => ({ query: `query-${i}` }), - ); - await expect(searchBatch(db, provider, requests)).rejects.toThrow( - `Batch size ${BATCH_SEARCH_MAX_REQUESTS + 1} exceeds maximum of ${BATCH_SEARCH_MAX_REQUESTS}`, - ); - }); - - it("should accept exactly the maximum number of requests", async () => { - const requests: BatchSearchRequest[] = Array.from( - { length: BATCH_SEARCH_MAX_REQUESTS }, - (_, i) => ({ query: `query-${i}` }), - ); - const result = await searchBatch(db, provider, requests); - expect(Object.keys(result.results)).toHaveLength(BATCH_SEARCH_MAX_REQUESTS); - }); - - it("should return results keyed by query string", async () => { - const requests: BatchSearchRequest[] = [{ query: "TypeScript" }, { query: "JavaScript" }]; - const result = await searchBatch(db, provider, requests); - - expect(result.results).toHaveProperty("TypeScript"); - expect(result.results).toHaveProperty("JavaScript"); - }); - - it("should execute a single request and return results", async () => { - const result = await searchBatch(db, provider, [{ query: "TypeScript" }]); - expect(Object.keys(result.results)).toHaveLength(1); - expect(result.results["TypeScript"]).toBeDefined(); - expect(result.results["TypeScript"]!.results).toBeDefined(); - }); - - it("should pass per-request options through to search", async () => { - const result = await searchBatch(db, provider, [ - { query: "TypeScript", options: { limit: 1 } }, - ]); - expect(result.results["TypeScript"]).toBeDefined(); - }); -}); diff --git a/tests/unit/batch.test.ts b/tests/unit/batch.test.ts deleted file mode 100644 index cb8980bb..00000000 --- a/tests/unit/batch.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { describe, it, expect, vi } from "vitest"; -import { batchImport } from "../../src/core/batch.js"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../../src/providers/embedding.js"; - -// Mock indexing module -vi.mock("../../src/core/indexing.js", () => ({ - indexDocument: vi.fn().mockResolvedValue({ id: "doc-1", chunkCount: 3 }), -})); - -// Mock fs -vi.mock("node:fs", () => ({ - readFileSync: vi.fn().mockReturnValue("# Test content\nSome text."), -})); - -// Mock logger -vi.mock("../../src/logger.js", () => ({ - getLogger: (): Record<string, ReturnType<typeof vi.fn>> => ({ - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - }), -})); - -function createMockDb(): Database.Database { - return {} as Database.Database; -} - -function createMockProvider(): EmbeddingProvider { - return { - name: "mock", - dimensions: 384, - embed: vi.fn().mockResolvedValue(new Array(384).fill(0)), - embedBatch: vi.fn().mockResolvedValue([new Array(384).fill(0)]), - }; -} - -describe("batchImport", () => { - it("should return empty results for no files", async () => { - const result = await batchImport(createMockDb(), createMockProvider(), []); - - expect(result.total).toBe(0); - expect(result.completed).toBe(0); - expect(result.failed).toBe(0); - expect(result.results).toEqual([]); - }); - - it("should process files and return results", async () => { - const files = ["file1.md", "file2.md", "file3.md"]; - const result = await batchImport(createMockDb(), createMockProvider(), files); - - expect(result.total).toBe(3); - expect(result.completed).toBe(3); - expect(result.failed).toBe(0); - expect(result.results).toHaveLength(3); - for (const r of result.results) { - expect(r.success).toBe(true); - expect(r.chunkCount).toBe(3); - expect(r.documentId).toBe("doc-1"); - } - }); - - it("should respect concurrency option", async () => { - const files = Array.from({ length: 10 }, (_, i) => `file${i}.md`); - const result = await batchImport(createMockDb(), createMockProvider(), files, { - concurrency: 2, - }); - - expect(result.total).toBe(10); - expect(result.completed).toBe(10); - }); - - it("should call onProgress callback", async () => { - const files = ["a.md", "b.md"]; - const progressCalls: Array<{ completed: number; failed: number }> = []; - - await batchImport(createMockDb(), createMockProvider(), files, { - onProgress: (progress) => { - progressCalls.push({ completed: progress.completed, failed: progress.failed }); - }, - }); - - expect(progressCalls.length).toBe(2); - const lastCall = progressCalls[progressCalls.length - 1]; - expect(lastCall.completed + lastCall.failed).toBe(2); - }); - - it("should handle failed files gracefully", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const mockedIndex = vi.mocked(indexDocument); - - mockedIndex.mockRejectedValueOnce(new Error("bad file")); - mockedIndex.mockResolvedValueOnce({ id: "doc-2", chunkCount: 1 }); - - const files = ["bad.md", "good.md"]; - const result = await batchImport(createMockDb(), createMockProvider(), files, { - concurrency: 1, - }); - - expect(result.total).toBe(2); - expect(result.completed).toBe(1); - expect(result.failed).toBe(1); - expect(result.results[0].success).toBe(false); - expect(result.results[0].error).toBe("bad file"); - expect(result.results[1].success).toBe(true); - }); - - it("should pass library and topic options through", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const mockedIndex = vi.mocked(indexDocument); - mockedIndex.mockResolvedValue({ id: "doc-3", chunkCount: 2 }); - - await batchImport(createMockDb(), createMockProvider(), ["test.md"], { - library: "react", - version: "18.0", - topicId: "frontend", - }); - - expect(mockedIndex).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - expect.objectContaining({ - library: "react", - version: "18.0", - topicId: "frontend", - sourceType: "library", - }), - ); - }); - - it("should default concurrency to 5", async () => { - const files = Array.from({ length: 8 }, (_, i) => `file${i}.md`); - const result = await batchImport(createMockDb(), createMockProvider(), files); - - expect(result.total).toBe(8); - expect(result.completed).toBe(8); - }); -}); diff --git a/tests/unit/bootstrap.test.ts b/tests/unit/bootstrap.test.ts new file mode 100644 index 00000000..370c2aae --- /dev/null +++ b/tests/unit/bootstrap.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { LibScopeConfig } from "../../src/config.js"; +import { bootstrap, mergeConfig, UnavailableEmbeddingProvider } from "../../src/core/bootstrap.js"; +import { getVectorTableDimensions } from "../../src/db/index-meta.js"; +import { ConfigError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; + +initLogger("silent"); + +function config(provider = "local"): LibScopeConfig { + return { + embedding: { provider }, + llm: {}, + database: {}, + indexing: { maxDocumentSize: 1000, allowPrivateUrls: false, allowSelfSignedCerts: false }, + logging: { level: "silent" }, + } as LibScopeConfig; +} + +class EightDimProvider extends MockEmbeddingProvider { + override readonly dimensions = 8 as 4; +} + +describe("bootstrap", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "libscope-bootstrap-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("opens and migrates the database and creates the vector table", () => { + const dbPath = join(dir, "kb.db"); + const app = bootstrap({ dbPath, config: config(), provider: new MockEmbeddingProvider() }); + expect(app.dbPath).toBe(dbPath); + const row = app.db.prepare("SELECT MAX(version) AS v FROM schema_version").get() as { + v: number; + }; + expect(row.v).toBeGreaterThan(0); + expect(getVectorTableDimensions(app.db)).toBe(4); + app.close(); + app.close(); + expect(app.db.open).toBe(false); + }); + + it("uses database.path from config when no dbPath is given", () => { + const dbPath = join(dir, "from-config.db"); + const app = bootstrap({ + config: config(), + configOverrides: { database: { path: dbPath } }, + provider: new MockEmbeddingProvider(), + }); + expect(app.dbPath).toBe(dbPath); + app.close(); + }); + + it("fails on an embedding model mismatch and closes the database it opened", () => { + const dbPath = join(dir, "kb.db"); + bootstrap({ dbPath, config: config(), provider: new MockEmbeddingProvider() }).close(); + expect(() => + bootstrap({ + dbPath, + config: config(), + provider: new EightDimProvider(), + vectorTable: "best-effort", + }), + ).toThrow(ConfigError); + }); + + it("keeps going without an embedding provider when it is not required", async () => { + const app = bootstrap({ + dbPath: join(dir, "kb.db"), + config: config("openai"), + requireProvider: false, + }); + expect(app.provider).toBeInstanceOf(UnavailableEmbeddingProvider); + await expect(app.provider.embed("x")).rejects.toBeInstanceOf(ConfigError); + expect(getVectorTableDimensions(app.db)).toBeUndefined(); + app.close(); + expect(() => bootstrap({ dbPath: join(dir, "kb2.db"), config: config("openai") })).toThrow( + ConfigError, + ); + }); + + it("uses an injected database without closing it", () => { + const db = createTestDb(); + const app = bootstrap({ + db, + config: config(), + provider: new MockEmbeddingProvider(), + vectorTable: "skip", + }); + app.close(); + expect(db.open).toBe(true); + db.close(); + }); +}); + +describe("mergeConfig", () => { + it("merges each section one level deep", () => { + const merged = mergeConfig(config(), { indexing: { allowPrivateUrls: true } }); + expect(merged.indexing).toEqual({ + maxDocumentSize: 1000, + allowPrivateUrls: true, + allowSelfSignedCerts: false, + }); + expect(mergeConfig(config())).toEqual(config()); + }); +}); diff --git a/tests/unit/chunk-writer.test.ts b/tests/unit/chunk-writer.test.ts new file mode 100644 index 00000000..0679b48a --- /dev/null +++ b/tests/unit/chunk-writer.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect, afterEach } from "vitest"; +import type Database from "better-sqlite3"; +import { createDatabase } from "../../src/db/connection.js"; +import { runMigrations, createVectorTable } from "../../src/db/schema.js"; +import { createChunkWriter } from "../../src/core/indexing.js"; +import { EmbeddingError } from "../../src/errors.js"; + +function hasSqliteVec(): boolean { + const db = createDatabase(":memory:"); + try { + db.prepare("SELECT vec_version()").get(); + return true; + } catch { + return false; + } finally { + db.close(); + } +} + +describe.skipIf(!hasSqliteVec())("createChunkWriter (sqlite-vec)", () => { + let db: Database.Database | undefined; + + afterEach(() => { + db?.close(); + db = undefined; + }); + + function openIndexedDb(): Database.Database { + const database = createDatabase(":memory:"); + runMigrations(database); + createVectorTable(database, 4); + database + .prepare("INSERT INTO documents (id, title, content, source_type) VALUES (?, ?, ?, ?)") + .run("doc-1", "Doc", "content", "manual"); + return database; + } + + it("stores chunks whose vectors match the index size", () => { + db = openIndexedDb(); + createChunkWriter(db).insertChunks("doc-1", ["a"], [[0.1, 0.2, 0.3, 0.4]]); + const row = db.prepare("SELECT COUNT(*) AS n FROM chunk_embeddings").get() as { n: number }; + expect(row.n).toBe(1); + }); + + it("explains how to rebuild when a vector does not match the index size", () => { + db = openIndexedDb(); + const writer = createChunkWriter(db); + let error: unknown; + try { + writer.insertChunks("doc-1", ["a"], [[0.1, 0.2, 0.3]]); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(EmbeddingError); + expect((error as Error).message).toMatch(/3 dimensions.*libscope admin reindex --rebuild/); + }); +}); diff --git a/tests/unit/cli-commands.test.ts b/tests/unit/cli-commands.test.ts new file mode 100644 index 00000000..c4e312aa --- /dev/null +++ b/tests/unit/cli-commands.test.ts @@ -0,0 +1,491 @@ +/** + * CLI contract tests: each command line maps to one operation with the expected input. + * runOperation is replaced by a spy that returns a canned result, and bootstrap() by an + * in-memory database, so nothing touches disk or the network. + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MockInstance } from "vitest"; +import type { Command } from "commander"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Operation } from "../../src/core/operations/index.js"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { testConfig } from "./operations/helpers.js"; + +const db = createTestDb(); + +vi.mock("../../src/core/bootstrap.js", () => ({ + bootstrap: (): unknown => ({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + dbPath: ":memory:", + close: (): void => {}, + }), +})); + +const CANNED: Record<string, unknown> = { + add: { kind: "file", documents: [], errors: [], skipped: [] }, + "get-document": { document: { title: "Doc" } }, + "bulk-delete": { affected: 2, documentIds: ["d1", "d2"] }, + "bulk-retag": { affected: 2, documentIds: ["d1", "d2"] }, + "bulk-move": { affected: 2, documentIds: ["d1", "d2"] }, + "create-pack": { name: "p", version: "1.0.0", documents: [] }, +}; + +vi.mock("../../src/core/operations/index.js", async (importOriginal) => { + const actual = await importOriginal<typeof import("../../src/core/operations/index.js")>(); + return { + ...actual, + runOperation: vi.fn( + (op: Operation): Promise<unknown> => Promise.resolve(CANNED[op.name] ?? { items: [] }), + ), + }; +}); + +const { runOperation } = await import("../../src/core/operations/index.js"); +const { main, program } = await import("../../src/cli/index.js"); + +/** Run `libscope --json <args>` and return [operation name, input] of each operation call. */ +async function invoke(args: string[]): Promise<Array<[string, unknown]>> { + vi.mocked(runOperation).mockClear(); + await main(["node", "libscope", "--json", ...args]); + expect(process.exitCode ?? 0).toBe(0); + return vi.mocked(runOperation).mock.calls.map(([op, , input]) => [op.name, input]); +} + +let home: string; +const savedHome = process.env["HOME"]; + +beforeAll(() => { + home = mkdtempSync(join(tmpdir(), "libscope-cli-")); + process.env["HOME"] = home; +}); + +afterAll(() => { + process.env["HOME"] = savedHome; + rmSync(home, { recursive: true, force: true }); +}); + +let stderrWrite: MockInstance; + +beforeEach(() => { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + stderrWrite = vi.spyOn(process.stderr, "write").mockImplementation(() => true); +}); + +afterEach(() => { + process.exitCode = undefined; + vi.restoreAllMocks(); +}); + +/** [argv, operation, expected input] for commands that call exactly one operation. */ +const SINGLE: Array<[string[], string, Record<string, unknown>]> = [ + [ + ["add", "a.md", "--topic", "t", "--library", "lib", "--lib-version", "1", "--tags", "x, y"], + "add", + { source: "a.md", topic: "t", library: "lib", version: "1", tags: ["x", "y"] }, + ], + [ + ["add", "https://e.com/docs", "--spider", "--max-pages", "5", "--no-same-domain"], + "add", + { source: "https://e.com/docs", spider: true, maxPages: 5, sameDomain: false }, + ], + [ + ["add", "https://e.com/d", "--spider", "--exclude-urls", "*.pdf", "--path-prefix", "/d"], + "add", + { spider: true, sameDomain: true, excludePatterns: ["*.pdf"], pathPrefix: "/d" }, + ], + [ + ["add", "https://github.com/o/r", "--branch", "dev", "--path", "docs,guide", "--token", "T"], + "add", + { source: "https://github.com/o/r", branch: "dev", paths: ["docs", "guide"], token: "T" }, + ], + [ + ["add", "dir", "--include", "**/*.md", "--exclude", "drafts/**", "--dry-run"], + "add", + { source: "dir", include: ["**/*.md"], exclude: ["drafts/**"], dryRun: true }, + ], + [ + [ + "add", + "a.md", + "--title", + "T", + "--format", + ".txt", + "--dedup", + "skip", + "--source-type", + "topic", + ], + "add", + { title: "T", format: ".txt", dedup: "skip", sourceType: "topic" }, + ], + [ + ["search", "how", "--topic", "t", "--source-type", "library", "--tags", "a", "-n", "3"], + "search", + { query: "how", topic: "t", sourceType: "library", tags: ["a"], limit: 3 }, + ], + [ + ["search", "q", "--offset", "2", "--min-rating", "4", "--max-per-doc", "1", "--context", "1"], + "search", + { query: "q", offset: 2, minRating: 4, maxChunksPerDocument: 1, contextChunks: 1 }, + ], + [["search", "--related", "doc-1", "-n", "4"], "search", { relatedTo: "doc-1", limit: 4 }], + [ + ["ask", "why?", "-n", "7", "--library", "lib", "--model", "m"], + "ask", + { question: "why?", topK: 7, library: "lib" }, + ], + [ + ["docs", "list", "--library", "lib", "-n", "5", "--offset", "10"], + "list-documents", + { library: "lib", limit: 5, offset: 10 }, + ], + [ + ["docs", "show", "d1", "--offset", "10", "--max-length", "100"], + "get-document", + { documentId: "d1", offset: 10, maxLength: 100 }, + ], + [ + ["docs", "update", "d1", "--title", "T", "--lib-version", "2", "--tags", "a,b", "--topic", "t"], + "update-document", + { documentId: "d1", title: "T", version: "2", tags: ["a", "b"], topic: "t" }, + ], + [["docs", "history", "d1"], "document-history", { documentId: "d1" }], + [["docs", "rollback", "d1", "3"], "rollback-document", { documentId: "d1", version: 3 }], + [ + ["docs", "rate", "d1", "4", "--feedback", "good", "--chunk", "c1"], + "rate-document", + { documentId: "d1", rating: 4, feedback: "good", chunkId: "c1" }, + ], + [ + ["docs", "link", "d1", "d2", "--type", "prerequisite", "--label", "first"], + "link-documents", + { documentId: "d1", targetDocumentId: "d2", linkType: "prerequisite", label: "first" }, + ], + [["docs", "unlink", "l1"], "unlink-documents", { linkId: "l1" }], + [ + ["docs", "links", "d1", "--type", "related"], + "list-links", + { documentId: "d1", linkType: "related" }, + ], + [["docs", "links"], "list-links", {}], + [["docs", "prereqs", "d1"], "prerequisites", { documentId: "d1" }], + [["docs", "tag", "d1", "a", "b,c"], "add-tags", { documentId: "d1", tags: ["a", "b", "c"] }], + [["docs", "untag", "d1", "a"], "remove-tags", { documentId: "d1", tags: ["a"] }], + [["docs", "suggest-tags", "d1", "-n", "3"], "suggest-tags", { documentId: "d1", limit: 3 }], + [["topics", "list", "--parent", "p"], "list-topics", { parent: "p" }], + [ + ["topics", "create", "n", "--description", "d", "--parent", "p"], + "create-topic", + { name: "n", description: "d", parent: "p" }, + ], + [ + ["topics", "delete", "t", "--delete-documents", "-y"], + "delete-topic", + { topic: "t", deleteDocuments: true }, + ], + [["tags", "list"], "list-tags", {}], + [ + ["searches", "save", "n", "q", "--topic", "t", "-n", "5", "--min-rating", "3"], + "save-search", + { name: "n", query: "q", topic: "t", limit: 5, minRating: 3 }, + ], + [ + ["searches", "list", "-n", "2", "--offset", "1"], + "list-saved-searches", + { limit: 2, offset: 1 }, + ], + [["searches", "run", "n"], "run-saved-search", { search: "n" }], + [["searches", "delete", "n"], "delete-saved-search", { search: "n" }], + [["sync", "conn"], "sync", { name: "conn" }], + [["sync", "--all"], "sync", { all: true }], + [ + ["disconnect", "conn", "--type", "slack", "--keep-documents", "-y"], + "disconnect", + { name: "conn", type: "slack", keepDocuments: true }, + ], + [["connections"], "list-connections", {}], + [ + [ + "pack", + "install", + "./p.json", + "--batch-size", + "5", + "--resume-from", + "2", + "--concurrency", + "3", + ], + "install-pack", + { pack: "./p.json", batchSize: 5, resumeFrom: 2, concurrency: 3 }, + ], + [["pack", "remove", "p", "-y"], "remove-pack", { pack: "p" }], + [ + ["pack", "install", "react-docs@1.2.0", "--registry", "official"], + "install-pack", + { pack: "react-docs@1.2.0", registry: "official" }, + ], + [["pack", "list"], "list-packs", {}], + [ + ["pack", "list", "--available", "--registry", "official"], + "list-packs", + { available: true, registry: "official" }, + ], + [ + ["registry", "add", "https://g.example/org/packs.git", "--name", "team", "--no-sync"], + "add-registry", + { url: "https://g.example/org/packs.git", name: "team", sync: false }, + ], + [["registry", "remove", "team", "-y"], "remove-registry", { name: "team" }], + [["registry", "list"], "list-registries", {}], + [["registry", "sync"], "sync-registries", {}], + [["registry", "sync", "team"], "sync-registries", { name: "team" }], + [ + ["registry", "search", "react", "--registry", "team"], + "search-registries", + { query: "react", registry: "team" }, + ], + [["registry", "create", "./reg"], "create-registry", { path: "./reg" }], + [ + [ + "registry", + "publish", + "p.json.gz", + "--registry", + "team", + "--pack-version", + "2.0.0", + "-m", + "msg", + "--submit", + ], + "publish-pack", + { file: "p.json.gz", registry: "team", version: "2.0.0", message: "msg", submit: true }, + ], + [ + ["registry", "unpublish", "p@1.0.0", "--registry", "team", "-y"], + "unpublish-pack", + { pack: "p@1.0.0", registry: "team" }, + ], + [ + ["pack", "create", "--name", "p", "--from", "./src", "--exclude", "*.min.js"], + "create-pack", + { name: "p", from: ["./src"], outputPath: "p.json.gz", exclude: ["*.min.js"], recursive: true }, + ], + [ + ["pack", "create", "--name", "p", "--topic", "t", "--pack-version", "2.0.0"], + "create-pack", + { name: "p", topic: "t", version: "2.0.0", outputPath: "p.json" }, + ], + [ + ["webhooks", "create", "https://h.example", "--events", "document.created,document.deleted"], + "create-webhook", + { url: "https://h.example", events: ["document.created", "document.deleted"] }, + ], + [["webhooks", "list"], "list-webhooks", {}], + [["webhooks", "delete", "w1"], "delete-webhook", { webhookId: "w1" }], + [["webhooks", "test", "w1"], "test-webhook", { webhookId: "w1" }], + [["admin", "reindex", "--rebuild"], "reindex", { rebuild: true }], + [ + ["admin", "reindex", "--doc", "d1", "d2", "--since", "2024-01-01", "--batch-size", "10"], + "reindex", + { documentIds: ["d1", "d2"], since: "2024-01-01", batchSize: 10 }, + ], + [ + ["admin", "dedupe", "--threshold", "0.9", "--strategy", "exact"], + "dedupe", + { threshold: 0.9, strategy: "exact" }, + ], + [["admin", "backup", "kb.json"], "backup", { outputPath: "kb.json" }], + [["admin", "restore", "kb.json", "-y"], "restore", { backupPath: "kb.json" }], + [["admin", "prune"], "prune-expired", {}], +]; + +describe("CLI command -> operation contract", () => { + it.each(SINGLE)("%j calls %s", async (argv, opName, expected) => { + const calls = await invoke(argv); + expect(calls).toHaveLength(1); + expect(calls[0]?.[0]).toBe(opName); + expect(calls[0]?.[1]).toMatchObject(expected); + }); + + it("leaves unset flags out so operation defaults apply", async () => { + const [[, input]] = (await invoke(["search", "q"])) as [[string, unknown]]; + expect(input).toEqual({ query: "q" }); + }); + + it("adds each source with its own operation call", async () => { + const calls = await invoke(["add", "a.md", "b.md", "--library", "lib"]); + expect(calls.map(([name, input]) => [name, (input as { source: string }).source])).toEqual([ + ["add", "a.md"], + ["add", "b.md"], + ]); + }); + + it("docs delete looks the document up, then deletes it", async () => { + const calls = await invoke(["docs", "delete", "d1", "-y"]); + expect(calls).toEqual([ + ["get-document", { documentId: "d1", maxLength: 0 }], + ["delete-document", { documentId: "d1" }], + ]); + }); + + it.each([ + [["bulk", "delete", "--topic", "t", "-y"], "bulk-delete", { topic: "t" }], + [ + ["bulk", "retag", "--tags", "old", "--add", "a,b", "--remove", "c", "-y"], + "bulk-retag", + { tags: ["old"], addTags: ["a", "b"], removeTags: ["c"] }, + ], + [ + ["bulk", "move", "--library", "lib", "--since", "2024-01-01", "--to", "t2", "-y"], + "bulk-move", + { library: "lib", dateFrom: "2024-01-01", targetTopic: "t2" }, + ], + ])("%j previews (dryRun) and then applies %s", async (argv, opName, expected) => { + const calls = await invoke(argv); + expect(calls.map(([name]) => name)).toEqual([opName, opName]); + expect(calls[0]?.[1]).toEqual({ ...expected, dryRun: true }); + expect(calls[1]?.[1]).toEqual(expected); + }); + + it("bulk --dry-run only previews", async () => { + const calls = await invoke(["bulk", "delete", "--topic", "t", "--dry-run"]); + expect(calls).toEqual([["bulk-delete", { topic: "t", dryRun: true }]]); + }); + + it("admin stats combines overview, popular, stale and search analytics", async () => { + const calls = await invoke(["admin", "stats", "--days", "7"]); + expect(calls).toEqual([ + ["overview", {}], + ["popular", {}], + ["stale", { days: 7 }], + ["search-analytics", { days: 7 }], + ]); + }); + + it("connect saves the connection (with schedule) and skips the sync with --no-sync", async () => { + const vault = join(home, "vault"); + const calls = await invoke([ + "connect", + "obsidian", + vault, + "--name", + "notes", + "--schedule", + "0 * * * *", + "--no-sync", + ]); + expect(calls).toEqual([]); + const saved = JSON.parse( + readFileSync(join(home, ".libscope", "connectors", "notes.json"), "utf-8"), + ) as Record<string, unknown>; + expect(saved).toMatchObject({ + connectorType: "obsidian", + vaultPath: vault, + topicMapping: "folder", + excludePatterns: [], + schedule: { cronExpression: "0 * * * *" }, + }); + + // Re-running changes only what is given; "off" removes the schedule; then it syncs. + const again = await invoke(["connect", "obsidian", "--name", "notes", "--schedule", "off"]); + expect(again).toEqual([["sync", { name: "notes" }]]); + const updated = JSON.parse( + readFileSync(join(home, ".libscope", "connectors", "notes.json"), "utf-8"), + ) as Record<string, unknown>; + expect(updated["schedule"]).toBeUndefined(); + expect(updated["vaultPath"]).toBe(vault); + }); + + it("connect rejects missing required settings", async () => { + await main(["node", "libscope", "connect", "slack", "--no-sync"]); + expect(process.exitCode).toBe(1); + expect(stderrWrite.mock.calls.join("")).toContain("Invalid Slack settings: token"); + }); +}); + +/** Every leaf command path, e.g. "docs show". */ +function leafCommands(cmd: Command, prefix: string[] = []): string[] { + return cmd.commands.flatMap((sub) => { + const path = [...prefix, sub.name()]; + return sub.commands.length > 0 ? leafCommands(sub, path) : [path.join(" ")]; + }); +} + +/** The command path an argv selects, e.g. ["docs", "show", "d1"] -> "docs show". */ +function commandPath(argv: string[]): string { + const path: string[] = []; + let cmd: Command = program; + for (const word of argv) { + const sub = cmd.commands.find((c) => c.name() === word); + if (!sub) break; + path.push(word); + cmd = sub; + } + return path.join(" "); +} + +describe("CLI command set", () => { + /** Commands not covered above: they do not call operations (config files, servers). */ + const NOT_OPERATIONS = new Set([ + "serve", + "doctor", + "config show", + "config get", + "config set", + "config unset", + "config path", + "workspace create", + "workspace list", + "workspace use", + "workspace delete", + ]); + + it("covers every command with a contract test", () => { + const tested = new Set([ + ...SINGLE.map(([argv]) => commandPath(argv)), + "docs delete", + "bulk delete", + "bulk retag", + "bulk move", + "admin stats", + "connect", + ]); + const missing = leafCommands(program).filter((c) => !NOT_OPERATIONS.has(c) && !tested.has(c)); + expect(missing).toEqual([]); + }); + + it.each([ + "init", + "import", + "import-batch", + "add-repo", + "repl", + "related", + "ratings", + "stats", + "search-analytics", + "watch", + "dedupe", + "reindex", + "export", + "import-backup", + "link", + "links", + "unlink", + "prereqs", + "tag", + "schedule", + "update", + ])("has no top-level %s command", (name) => { + expect(program.commands.map((c) => c.name())).not.toContain(name); + }); +}); diff --git a/tests/unit/cli-connect-settings.test.ts b/tests/unit/cli-connect-settings.test.ts new file mode 100644 index 00000000..e7d9cc14 --- /dev/null +++ b/tests/unit/cli-connect-settings.test.ts @@ -0,0 +1,159 @@ +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { connectSettings, type ConnectFlags } from "../../src/cli/commands/connectors.js"; +import { ValidationError } from "../../src/errors.js"; + +const noFlags: ConnectFlags = { sync: true }; + +describe("connectSettings", () => { + it("notion passes the token and excluded pages", () => { + expect( + connectSettings("notion", undefined, { sync: true, token: "t", exclude: "a, b" }, true), + ).toEqual({ + token: "t", + excludePages: ["a", "b"], + }); + }); + + it("slack applies defaults only to a new connection", () => { + expect(connectSettings("slack", undefined, noFlags, true)).toEqual({ + token: undefined, + channels: ["all"], + excludeChannels: undefined, + threadMode: "aggregate", + }); + expect(connectSettings("slack", undefined, noFlags, false)).toEqual({ + token: undefined, + channels: undefined, + excludeChannels: undefined, + threadMode: undefined, + }); + expect( + connectSettings( + "slack", + undefined, + { sync: true, channels: "x", threadMode: "separate" }, + true, + ), + ).toMatchObject({ channels: ["x"], threadMode: "separate" }); + }); + + it("confluence needs a base URL for a new connection", () => { + expect(() => connectSettings("confluence", undefined, noFlags, true)).toThrow(ValidationError); + expect(connectSettings("confluence", "https://c.example", noFlags, true)).toEqual({ + baseUrl: "https://c.example", + type: "cloud", + email: undefined, + token: undefined, + spaces: ["all"], + excludeSpaces: undefined, + }); + }); + + it("confluence keeps saved values for an existing connection", () => { + expect(connectSettings("confluence", undefined, noFlags, false)).toEqual({ + baseUrl: undefined, + type: undefined, + email: undefined, + token: undefined, + spaces: undefined, + excludeSpaces: undefined, + }); + }); + + it.each([true, false])("confluence --server sets the server type (new: %s)", (isNew) => { + const settings = connectSettings( + "confluence", + "https://c.example", + { sync: true, server: true }, + isNew, + ); + expect(settings["type"]).toBe("server"); + }); + + it("obsidian resolves the vault path and applies defaults to a new connection", () => { + expect(connectSettings("obsidian", "vault", noFlags, true)).toEqual({ + vaultPath: resolve("vault"), + topicMapping: "folder", + excludePatterns: [], + }); + expect(() => connectSettings("obsidian", undefined, noFlags, true)).toThrow(ValidationError); + }); + + it("obsidian keeps saved values for an existing connection", () => { + expect(connectSettings("obsidian", undefined, noFlags, false)).toEqual({ + vaultPath: undefined, + topicMapping: undefined, + excludePatterns: undefined, + }); + expect(connectSettings("obsidian", "", noFlags, false)["vaultPath"]).toBe(""); + expect( + connectSettings("obsidian", "v2", { sync: true, topicMapping: "tags", exclude: "x" }, false), + ).toEqual({ vaultPath: resolve("v2"), topicMapping: "tags", excludePatterns: ["x"] }); + }); + + it("onenote applies defaults to a new connection", () => { + expect(connectSettings("onenote", undefined, noFlags, true)).toEqual({ + clientId: "", + tenantId: "common", + notebooks: ["all"], + excludeSections: [], + }); + }); + + it("onenote keeps saved values for an existing connection", () => { + expect(connectSettings("onenote", undefined, noFlags, false)).toEqual({ + clientId: undefined, + tenantId: undefined, + notebooks: undefined, + excludeSections: undefined, + }); + }); + + it.each([true, false])("onenote --notebook and --token override (new: %s)", (isNew) => { + expect( + connectSettings( + "onenote", + undefined, + { sync: true, notebook: "Work", token: "tok", clientId: "c", tenantId: "t" }, + isNew, + ), + ).toMatchObject({ + clientId: "c", + tenantId: "t", + notebooks: ["Work"], + accessToken: "tok", + refreshToken: undefined, + tokenExpiry: undefined, + }); + }); + + it("docs needs a site URL for a new connection and passes the crawl flags", () => { + expect(() => connectSettings("docs", undefined, noFlags, true)).toThrow(ValidationError); + expect( + connectSettings( + "docs", + "https://docs.example", + { + sync: true, + siteType: "sphinx", + library: "lib", + libVersion: "1", + maxPages: 5, + maxDepth: 2, + pathPrefix: "/p", + }, + true, + ), + ).toEqual({ + url: "https://docs.example", + type: "sphinx", + library: "lib", + version: "1", + maxPages: 5, + maxDepth: 2, + pathPrefix: "/p", + }); + expect(connectSettings("docs", undefined, noFlags, false)["url"]).toBeUndefined(); + }); +}); diff --git a/tests/unit/cli-run.test.ts b/tests/unit/cli-run.test.ts new file mode 100644 index 00000000..a6217e45 --- /dev/null +++ b/tests/unit/cli-run.test.ts @@ -0,0 +1,160 @@ +/** + * CLI end-to-end over an in-memory database: add -> search -> docs show, --json output, + * and the one error handler (message, hint, stack only with --verbose, exit code 1). + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ConfigError, DocumentNotFoundError, ValidationError } from "../../src/errors.js"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { testConfig } from "./operations/helpers.js"; + +const db = createTestDbWithVec(); + +vi.mock("../../src/core/bootstrap.js", () => ({ + bootstrap: (): unknown => ({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + dbPath: ":memory:", + close: (): void => {}, + }), +})); + +const { main } = await import("../../src/cli/index.js"); +const { errorHint, formatError } = await import("../../src/cli/errors.js"); + +let dir: string; +let stdout: string[]; +let stderr: string[]; + +async function cli(...args: string[]): Promise<{ out: string; err: string; code: number }> { + stdout = []; + stderr = []; + process.exitCode = undefined; + await main(["node", "libscope", ...args]); + const code = Number(process.exitCode ?? 0); + process.exitCode = undefined; + return { out: stdout.join("\n"), err: stderr.join(""), code }; +} + +beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), "libscope-cli-run-")); + writeFileSync( + join(dir, "widgets.md"), + "# Widget Guide\n\nWidgets are configured with widget.yaml. Set the color key to change the color.", + ); +}); + +afterAll(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +beforeEach(() => { + vi.spyOn(console, "log").mockImplementation((...args: unknown[]) => { + stdout.push(args.map(String).join(" ")); + }); + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array) => { + stderr.push(String(chunk)); + return true; + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("CLI over a database", () => { + let documentId = ""; + + it("add prints the new document ID; --json prints the operation results", async () => { + const human = await cli("add", join(dir, "widgets.md"), "--tags", "guide"); + expect(human.code).toBe(0); + expect(human.out).toMatch(/✓ widgets \(1 chunk\) {2}[0-9a-f-]{36}/); + + // Adding the same file again updates the same document (the path is its URL). + const json = await cli("--json", "add", join(dir, "widgets.md")); + const results = JSON.parse(json.out) as Array<{ documents: Array<{ documentId: string }> }>; + documentId = results[0]?.documents[0]?.documentId ?? ""; + expect(human.out).toContain(documentId); + }); + + it("search shows document and chunk IDs; --json returns the ListResult", async () => { + const human = await cli("search", "widget color"); + expect(human.out).toContain(`document ${documentId} chunk `); + + const json = await cli("--json", "search", "widget color", "-n", "1"); + const result = JSON.parse(json.out) as { + items: Array<{ documentId: string; chunkId: string }>; + total: number; + limit: number; + }; + expect(result.limit).toBe(1); + expect(result.items[0]?.documentId).toBe(documentId); + expect(result.items[0]?.chunkId).toBeTruthy(); + }); + + it("docs show prints metadata, tags and content", async () => { + const { out, code } = await cli("docs", "show", documentId); + expect(code).toBe(0); + expect(out).toContain(`ID: ${documentId}`); + expect(out).toContain("Tags: guide"); + expect(out).toContain("Widgets are configured"); + }); + + it("docs list shows IDs", async () => { + const { out } = await cli("docs", "list"); + expect(out).toContain(`${documentId} widgets`); + }); + + it("ask without an LLM fails with the configuration hint", async () => { + const { err, code } = await cli("ask", "what configures widgets?"); + expect(code).toBe(1); + expect(err).toContain("✗ No LLM provider configured"); + expect(err).toContain("libscope doctor"); + }); + + it("not found: message, hint, no stack, exit code 1", async () => { + const { err, code } = await cli("docs", "show", "nope"); + expect(code).toBe(1); + expect(err).toContain("✗ Document not found: nope"); + expect(err).toContain("libscope docs list"); + expect(err).not.toContain(" at "); + }); + + it("--verbose adds the stack trace", async () => { + const { err } = await cli("--verbose", "docs", "show", "nope"); + expect(err).toContain("DocumentNotFoundError"); + expect(err).toContain(" at "); + }); + + it("invalid input is a validation error with a --help hint", async () => { + const { err, code } = await cli("docs", "rate", documentId, "9"); + expect(code).toBe(1); + expect(err).toContain("✗ Invalid input for rate-document"); + expect(err).toContain("--help"); + }); + + it("search without a query outside a terminal asks for one", async () => { + const { err, code } = await cli("--json", "search"); + expect(code).toBe(1); + expect(err).toContain("Give a search query"); + }); +}); + +describe("error hints", () => { + it("suggests a next step per error type", () => { + expect(errorHint(new DocumentNotFoundError("x"))).toContain("docs list"); + expect(errorHint(new ConfigError("x"))).toContain("doctor"); + expect(errorHint(new ValidationError("x"))).toContain("--help"); + expect(errorHint(new Error("x"))).toBeUndefined(); + }); + + it("formats the message first and the stack only when verbose", () => { + const err = new Error("boom"); + expect(formatError(err, false)).toEqual(["✗ boom"]); + expect(formatError(err, true).join("\n")).toContain("Error: boom"); + }); +}); diff --git a/tests/unit/cli-version-options.test.ts b/tests/unit/cli-version-options.test.ts new file mode 100644 index 00000000..de71ae65 --- /dev/null +++ b/tests/unit/cli-version-options.test.ts @@ -0,0 +1,84 @@ +import { describe, it, expect, vi, beforeAll } from "vitest"; +import { Command, CommanderError } from "commander"; +import { readFileSync } from "node:fs"; + +import { program } from "../../src/cli/index.js"; + +const pkgVersion = ( + JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf-8")) as { + version: string; + } +).version; + +/** Make every command throw instead of exiting, and swallow help/version output. */ +function makeTestable(cmd: Command, output: string[]): void { + cmd.exitOverride(); + cmd.configureOutput({ + writeOut: (str) => output.push(str), + writeErr: (str) => output.push(str), + }); + for (const sub of cmd.commands) makeTestable(sub, output); +} + +function findCommand(root: Command, path: string[]): Command { + let cmd = root; + for (const name of path) { + const next = cmd.commands.find((c) => c.name() === name); + if (!next) throw new Error(`Command not found: ${path.join(" ")}`); + cmd = next; + } + return cmd; +} + +/** Parse argv against the real CLI program with the target command's action stubbed out. */ +async function parseWithStubbedAction( + path: string[], + args: string[], +): Promise<Record<string, unknown>> { + const cmd = findCommand(program, path); + const action = vi.fn(); + cmd.action(action); + await program.parseAsync([...path, ...args], { from: "user" }); + expect(action).toHaveBeenCalledTimes(1); + return cmd.opts(); +} + +describe("CLI subcommand version flags", () => { + const output: string[] = []; + + beforeAll(() => { + makeTestable(program, output); + }); + + it("keeps the root --version flag printing the CLI version", async () => { + output.length = 0; + await expect(program.parseAsync(["--version"], { from: "user" })).rejects.toBeInstanceOf( + CommanderError, + ); + expect(output.join("")).toContain(pkgVersion); + }); + + it.each([ + [["add"], ["doc.md"]], + [["search"], ["query"]], + [["docs", "list"], []], + [["docs", "update"], ["doc-1"]], + ])("%j accepts --lib-version", async (path, positional) => { + const opts = await parseWithStubbedAction(path, [...positional, "--lib-version", "2.1.0"]); + expect(opts.libVersion).toBe("2.1.0"); + }); + + it.each([ + [ + ["pack", "create"], + ["--name", "react-docs"], + ], + [ + ["registry", "publish"], + ["pack.json", "--registry", "main"], + ], + ])("%j accepts --pack-version", async (path, rest) => { + const opts = await parseWithStubbedAction(path, [...rest, "--pack-version", "3.0.0"]); + expect(opts.packVersion).toBe("3.0.0"); + }); +}); diff --git a/tests/unit/code-chunker-grammar.test.ts b/tests/unit/code-chunker-grammar.test.ts new file mode 100644 index 00000000..f2e84e64 --- /dev/null +++ b/tests/unit/code-chunker-grammar.test.ts @@ -0,0 +1,29 @@ +import { describe, it, expect, vi } from "vitest"; +import { TreeSitterChunker } from "../../src/lite/chunker-treesitter.js"; + +// The grammar packages are CommonJS. import() of them yields a namespace whose `default` is +// module.exports, with no named exports (tree-sitter-typescript's are not statically detectable). +const typescriptLanguage = { name: "typescript" }; +const tsxLanguage = { name: "tsx" }; +const pythonLanguage = { name: "python", language: {} }; + +vi.mock("tree-sitter-typescript", () => ({ + default: { typescript: typescriptLanguage, tsx: tsxLanguage }, +})); +vi.mock("tree-sitter-python", () => ({ default: pythonLanguage })); + +interface GrammarLoader { + loadGrammar(language: string): Promise<unknown>; +} + +describe("TreeSitterChunker grammar loading", () => { + it("picks the typescript language out of tree-sitter-typescript's default export", async () => { + const loader = new TreeSitterChunker() as unknown as GrammarLoader; + await expect(loader.loadGrammar("typescript")).resolves.toBe(typescriptLanguage); + }); + + it("uses the default export of other grammar packages", async () => { + const loader = new TreeSitterChunker() as unknown as GrammarLoader; + await expect(loader.loadGrammar("python")).resolves.toBe(pythonLanguage); + }); +}); diff --git a/tests/unit/config-layers.test.ts b/tests/unit/config-layers.test.ts new file mode 100644 index 00000000..fdac032f --- /dev/null +++ b/tests/unit/config-layers.test.ts @@ -0,0 +1,342 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + loadConfig, + invalidateConfigCache, + maskSecret, + maskConfigSecrets, + getConfigKeyTable, +} from "../../src/config.js"; +import * as loggerModule from "../../src/logger.js"; +import { initLogger } from "../../src/logger.js"; + +/** Env vars that influence config loading; cleared for every test. */ +const CONFIG_ENV_VARS = [ + ...getConfigKeyTable().flatMap((row) => row.env), + "HOME", + "LIBSCOPE_WORKSPACE", +]; + +describe("config layering (temp HOME and cwd)", () => { + let tempHome: string; + let tempCwd: string; + const savedEnv: Record<string, string | undefined> = {}; + + function writeUserConfig(data: unknown): void { + mkdirSync(join(tempHome, ".libscope"), { recursive: true }); + writeFileSync(join(tempHome, ".libscope", "config.json"), JSON.stringify(data), "utf-8"); + } + + function writeSecrets(data: unknown): void { + mkdirSync(join(tempHome, ".libscope"), { recursive: true }); + writeFileSync(join(tempHome, ".libscope", "secrets.json"), JSON.stringify(data), "utf-8"); + } + + function writeProjectConfig(data: unknown): void { + writeFileSync(join(tempCwd, ".libscope.json"), JSON.stringify(data), "utf-8"); + } + + function load(env: Record<string, string> = {}): ReturnType<typeof loadConfig> { + Object.assign(process.env, env); + invalidateConfigCache(); + return loadConfig(); + } + + /** Replace the logger with a spy and return the warn mock. */ + function spyWarn(): ReturnType<typeof vi.fn> { + const warn = vi.fn(); + vi.spyOn(loggerModule, "getLogger").mockReturnValue({ + warn, + } as unknown as ReturnType<typeof loggerModule.getLogger>); + return warn; + } + + function messages(warn: ReturnType<typeof vi.fn>): string[] { + return warn.mock.calls.map((c) => String(c[0])); + } + + beforeEach(() => { + initLogger("silent"); + for (const key of CONFIG_ENV_VARS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + tempHome = mkdtempSync(join(tmpdir(), "libscope-config-home-")); + tempCwd = mkdtempSync(join(tmpdir(), "libscope-config-cwd-")); + process.env["HOME"] = tempHome; + vi.spyOn(process, "cwd").mockReturnValue(tempCwd); + }); + + afterEach(() => { + vi.restoreAllMocks(); + for (const [key, val] of Object.entries(savedEnv)) { + if (val === undefined) delete process.env[key]; + else process.env[key] = val; + } + invalidateConfigCache(); + rmSync(tempHome, { recursive: true, force: true }); + rmSync(tempCwd, { recursive: true, force: true }); + }); + + describe("defaults", () => { + it("applies schema defaults when no files and no env vars exist", () => { + const config = load(); + expect(config.embedding).toEqual({ provider: "local" }); + expect(config.llm).toEqual({ provider: "auto" }); + expect(config.indexing).toEqual({ + maxDocumentSize: 100 * 1024 * 1024, + allowPrivateUrls: false, + allowSelfSignedCerts: false, + }); + expect(config.logging.level).toBe("info"); + expect(config.mcp).toEqual({ toolsets: [] }); + expect(config.database).toEqual({}); + expect(config.openai).toBeUndefined(); + expect(config.anthropic).toBeUndefined(); + }); + }); + + describe("env overrides only replace keys that are set", () => { + it("keeps provider=openai from the config file when LIBSCOPE_OPENAI_API_KEY is set", () => { + writeUserConfig({ embedding: { provider: "openai" } }); + const config = load({ LIBSCOPE_OPENAI_API_KEY: "sk-env" }); + expect(config.embedding.provider).toBe("openai"); + expect(config.openai?.apiKey).toBe("sk-env"); + }); + + it("keeps file model/url when LIBSCOPE_EMBEDDING_PROVIDER is set", () => { + writeUserConfig({ + embedding: { url: "http://gpu-box:11434", model: "mxbai-embed-large" }, + }); + const config = load({ LIBSCOPE_EMBEDDING_PROVIDER: "ollama" }); + expect(config.embedding).toEqual({ + provider: "ollama", + url: "http://gpu-box:11434", + model: "mxbai-embed-large", + }); + }); + + it("keeps project-level indexing settings when one indexing env var is set", () => { + writeProjectConfig({ indexing: { allowPrivateUrls: true, maxDocumentSize: 1234 } }); + const config = load({ LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS: "1" }); + expect(config.indexing.allowSelfSignedCerts).toBe(true); + expect(config.indexing.allowPrivateUrls).toBe(true); + expect(config.indexing.maxDocumentSize).toBe(1234); + }); + + it("keeps file llm.model when only LIBSCOPE_LLM_PROVIDER is set", () => { + writeUserConfig({ llm: { provider: "ollama", model: "llama3" } }); + const config = load({ LIBSCOPE_LLM_PROVIDER: "openai" }); + expect(config.llm?.provider).toBe("openai"); + expect(config.llm?.model).toBe("llama3"); + }); + + it("project config wins over user config", () => { + writeUserConfig({ embedding: { model: "user-model" } }); + writeProjectConfig({ embedding: { model: "project-model" } }); + expect(load().embedding.model).toBe("project-model"); + }); + + it("ignores an invalid LIBSCOPE_EMBEDDING_PROVIDER without resetting the file value", () => { + const warn = spyWarn(); + writeUserConfig({ embedding: { provider: "ollama" } }); + const config = load({ LIBSCOPE_EMBEDDING_PROVIDER: "bogus" }); + expect(config.embedding.provider).toBe("ollama"); + expect(messages(warn).some((m) => m.includes("LIBSCOPE_EMBEDDING_PROVIDER"))).toBe(true); + }); + + it("an env value of false overrides true in a config file", () => { + writeUserConfig({ indexing: { allowPrivateUrls: true } }); + const config = load({ LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS: "false" }); + expect(config.indexing.allowPrivateUrls).toBe(false); + }); + }); + + describe("env var names are LIBSCOPE_<SECTION>_<FIELD>", () => { + it("reads every non-secret key from its env var", () => { + const config = load({ + LIBSCOPE_EMBEDDING_PROVIDER: "ollama", + LIBSCOPE_EMBEDDING_MODEL: "bge-m3", + LIBSCOPE_EMBEDDING_URL: "http://embed:11434", + LIBSCOPE_EMBEDDING_DIMENSIONS: "1024", + LIBSCOPE_LLM_PROVIDER: "ollama", + LIBSCOPE_LLM_MODEL: "llama3", + LIBSCOPE_LLM_URL: "http://llm:11434", + LIBSCOPE_INDEXING_MAX_DOCUMENT_SIZE: "2048", + LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS: "true", + LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS: "1", + LIBSCOPE_LOGGING_LEVEL: "warn", + }); + expect(config.embedding).toEqual({ + provider: "ollama", + model: "bge-m3", + url: "http://embed:11434", + dimensions: 1024, + }); + expect(config.llm).toEqual({ provider: "ollama", model: "llama3", url: "http://llm:11434" }); + expect(config.indexing).toEqual({ + maxDocumentSize: 2048, + allowPrivateUrls: true, + allowSelfSignedCerts: true, + }); + expect(config.logging.level).toBe("warn"); + }); + + it("LIBSCOPE_DATABASE_PATH sets database.path and expands ~", () => { + const config = load({ LIBSCOPE_DATABASE_PATH: "~/data/x.db" }); + expect(config.database.path).toBe(join(tempHome, "data", "x.db")); + }); + + it("old env var names are not read", () => { + const config = load({ + LIBSCOPE_OLLAMA_URL: "http://old:11434", + LIBSCOPE_OLLAMA_MODEL: "old-model", + LIBSCOPE_ALLOW_PRIVATE_URLS: "true", + }); + delete process.env["LIBSCOPE_OLLAMA_URL"]; + delete process.env["LIBSCOPE_OLLAMA_MODEL"]; + delete process.env["LIBSCOPE_ALLOW_PRIVATE_URLS"]; + expect(config.embedding.url).toBeUndefined(); + expect(config.embedding.model).toBeUndefined(); + expect(config.indexing.allowPrivateUrls).toBe(false); + }); + + it("ignores an invalid env value, logs a warning, and keeps the file value", () => { + const warn = spyWarn(); + writeUserConfig({ embedding: { dimensions: 512 } }); + const config = load({ LIBSCOPE_EMBEDDING_DIMENSIONS: "lots" }); + expect(config.embedding.dimensions).toBe(512); + expect(messages(warn).some((m) => m.includes("LIBSCOPE_EMBEDDING_DIMENSIONS"))).toBe(true); + }); + }); + + describe("API keys: LIBSCOPE_<P>_API_KEY > <P>_API_KEY > secrets.json", () => { + it("OPENAI_API_KEY alone sets openai.apiKey", () => { + expect(load({ OPENAI_API_KEY: "sk-plain" }).openai?.apiKey).toBe("sk-plain"); + }); + + it("LIBSCOPE_OPENAI_API_KEY wins over OPENAI_API_KEY", () => { + const config = load({ OPENAI_API_KEY: "sk-plain", LIBSCOPE_OPENAI_API_KEY: "sk-scoped" }); + expect(config.openai?.apiKey).toBe("sk-scoped"); + }); + + it("an empty LIBSCOPE_OPENAI_API_KEY falls through to OPENAI_API_KEY", () => { + const config = load({ LIBSCOPE_OPENAI_API_KEY: "", OPENAI_API_KEY: "sk-plain" }); + expect(config.openai?.apiKey).toBe("sk-plain"); + }); + + it("LIBSCOPE_ANTHROPIC_API_KEY wins over ANTHROPIC_API_KEY", () => { + const config = load({ + ANTHROPIC_API_KEY: "sk-ant-plain", + LIBSCOPE_ANTHROPIC_API_KEY: "sk-ant-scoped", + }); + expect(config.anthropic?.apiKey).toBe("sk-ant-scoped"); + }); + + it("secrets.json is used when no env key is set", () => { + writeSecrets({ openai: { apiKey: "sk-file" }, anthropic: { apiKey: "sk-ant-file" } }); + const config = load(); + expect(config.openai?.apiKey).toBe("sk-file"); + expect(config.anthropic?.apiKey).toBe("sk-ant-file"); + }); + + it("env keys win over secrets.json", () => { + writeSecrets({ openai: { apiKey: "sk-file" } }); + expect(load({ OPENAI_API_KEY: "sk-env" }).openai?.apiKey).toBe("sk-env"); + }); + + it("secrets.json holds only secrets: other keys there are ignored", () => { + writeSecrets({ logging: { level: "debug" } }); + expect(load().logging.level).toBe("info"); + }); + + it("API keys in config.json or .libscope.json are ignored with a warning", () => { + const warn = spyWarn(); + writeUserConfig({ openai: { apiKey: "sk-user-file" } }); + writeProjectConfig({ anthropic: { apiKey: "sk-ant-project-file" } }); + const config = load(); + expect(config.openai).toBeUndefined(); + expect(config.anthropic).toBeUndefined(); + const keyWarnings = messages(warn).filter((m) => m.includes("API keys are read only")); + expect(keyWarnings).toHaveLength(2); + expect(keyWarnings.join("\n")).toContain(join(tempCwd, ".libscope.json")); + expect(keyWarnings.join("\n")).toContain("secrets.json"); + }); + + it("validation accepts exactly the keys providers will use", () => { + const warn = spyWarn(); + writeUserConfig({ embedding: { provider: "openai" }, llm: { provider: "anthropic" } }); + load({ OPENAI_API_KEY: "sk-plain", ANTHROPIC_API_KEY: "sk-ant" }); + expect(warn).not.toHaveBeenCalled(); + }); + }); + + describe("config file validation", () => { + it("ignores an invalid file value, logs a warning, and keeps the default", () => { + const warn = spyWarn(); + writeUserConfig({ logging: { level: "loud" }, indexing: { maxDocumentSize: -1 } }); + const config = load(); + expect(config.logging.level).toBe("info"); + expect(config.indexing.maxDocumentSize).toBe(100 * 1024 * 1024); + expect(messages(warn).filter((m) => m.includes("invalid config value"))).toHaveLength(2); + }); + + it("accepts string booleans and integers in a file", () => { + writeUserConfig({ indexing: { allowSelfSignedCerts: "true", maxDocumentSize: "4096" } }); + const config = load(); + expect(config.indexing.allowSelfSignedCerts).toBe(true); + expect(config.indexing.maxDocumentSize).toBe(4096); + }); + + it("reads mcp.toolsets as an array or a comma list, and from LIBSCOPE_MCP_TOOLSETS", () => { + writeUserConfig({ mcp: { toolsets: ["admin"] } }); + expect(load().mcp?.toolsets).toEqual(["admin"]); + writeUserConfig({ mcp: { toolsets: "core, all" } }); + expect(load().mcp?.toolsets).toEqual(["core", "all"]); + expect(load({ LIBSCOPE_MCP_TOOLSETS: "admin" }).mcp?.toolsets).toEqual(["admin"]); + }); + + it("ignores an unknown mcp toolset with a warning", () => { + const warn = spyWarn(); + expect(load({ LIBSCOPE_MCP_TOOLSETS: "admin,bogus" }).mcp?.toolsets).toEqual([]); + expect(messages(warn).some((m) => m.includes("LIBSCOPE_MCP_TOOLSETS"))).toBe(true); + }); + + it("ignores an unknown embedding provider name in a file", () => { + writeUserConfig({ embedding: { provider: "my-plugin" } }); + expect(load().embedding.provider).toBe("local"); + }); + + it("drops unknown and old keys from the loaded config", () => { + writeUserConfig({ + embedding: { ollamaUrl: "http://old:11434", ollamaModel: "old" }, + registries: [], + }); + const config = load(); + expect(config.embedding).toEqual({ provider: "local" }); + expect(config).not.toHaveProperty("registries"); + }); + }); + + describe("secret masking", () => { + it("masks long keys to prefix and last four characters", () => { + expect(maskSecret("sk-abcdefghijklmnop1234")).toBe("sk-…1234"); + }); + + it("fully masks short keys", () => { + expect(maskSecret("short")).toBe("****"); + }); + + it("masks every API key in a config without mutating the original", () => { + writeSecrets({ anthropic: { apiKey: "sk-ant-secret-value-8888" } }); + const config = load({ LIBSCOPE_OPENAI_API_KEY: "sk-openai-secret-value-9999" }); + const masked = maskConfigSecrets(config); + expect(JSON.stringify(masked)).not.toContain("secret-value"); + expect(masked.openai?.apiKey).toBe("sk-…9999"); + expect(masked.anthropic?.apiKey).toBe("sk-…8888"); + expect(config.openai?.apiKey).toBe("sk-openai-secret-value-9999"); + }); + }); +}); diff --git a/tests/unit/config-save.test.ts b/tests/unit/config-save.test.ts index f29867ee..a1c6439b 100644 --- a/tests/unit/config-save.test.ts +++ b/tests/unit/config-save.test.ts @@ -1,10 +1,17 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdirSync, readFileSync, rmSync } from "node:fs"; +import { + existsSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, + chmodSync, +} from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { initLogger } from "../../src/logger.js"; -import type { LibScopeConfig } from "../../src/config.js"; // Create a unique temp HOME for each test run — must be initialized before module load let tempHome: string = join(tmpdir(), `libscope-config-save-test-${process.pid}`); @@ -19,89 +26,302 @@ vi.mock("node:os", async (importOriginal) => { }); // Dynamic import after mock is set up -const { saveUserConfig, invalidateConfigCache } = await import("../../src/config.js"); +const { + invalidateConfigCache, + setUserConfigValue, + unsetUserConfigValue, + getConfigValue, + getConfigFileFor, + getUserConfigPath, + getSecretsPath, + loadConfig, + CONFIG_KEY_NAMES, + getConfigKeyTable, +} = await import("../../src/config.js"); +const { saveRegistries, loadRegistries } = await import("../../src/registry/config.js"); -function readSavedConfig(): LibScopeConfig { - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); - return JSON.parse(written) as LibScopeConfig; +/** Env vars that would leak into loadConfig from the developer's shell. */ +const ENV_VARS = getConfigKeyTable().flatMap((row) => row.env); +const savedEnv: Record<string, string | undefined> = {}; + +function configPath(): string { + return join(tempHome, ".libscope", "config.json"); +} + +function secretsPath(): string { + return join(tempHome, ".libscope", "secrets.json"); +} + +function readJson(path: string): Record<string, unknown> { + return JSON.parse(readFileSync(path, "utf-8")) as Record<string, unknown>; +} + +function readSavedRaw(): Record<string, unknown> { + return readJson(configPath()); } -describe("saveUserConfig credential stripping", () => { - beforeEach(() => { - initLogger("silent"); - tempHome = join(tmpdir(), `libscope-config-save-test-${randomUUID()}`); - mkdirSync(tempHome, { recursive: true }); - invalidateConfigCache(); +function writeExisting(data: unknown, path = configPath()): void { + mkdirSync(join(tempHome, ".libscope"), { recursive: true }); + writeFileSync(path, JSON.stringify(data), "utf-8"); +} + +const REGISTRY = { + name: "team", + url: "https://example.com/registry.git", + lastSyncedAt: null, +}; + +beforeEach(() => { + initLogger("silent"); + for (const key of ENV_VARS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + tempHome = join(tmpdir(), `libscope-config-save-test-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + invalidateConfigCache(); +}); + +afterEach(() => { + for (const [key, val] of Object.entries(savedEnv)) { + if (val === undefined) delete process.env[key]; + else process.env[key] = val; + } + rmSync(tempHome, { recursive: true, force: true }); +}); + +describe("config keys", () => { + it("are derived from the schema", () => { + expect([...CONFIG_KEY_NAMES].sort()).toEqual( + [ + "anthropic.apiKey", + "database.path", + "embedding.dimensions", + "embedding.model", + "embedding.provider", + "embedding.url", + "indexing.allowPrivateUrls", + "indexing.allowSelfSignedCerts", + "indexing.maxDocumentSize", + "llm.model", + "llm.provider", + "llm.url", + "logging.level", + "mcp.toolsets", + "openai.apiKey", + ].sort(), + ); }); - afterEach(() => { - try { - rmSync(tempHome, { recursive: true, force: true }); - } catch { - // ignore cleanup errors - } + it("key table lists type, default, env vars, secret flag, and description", () => { + const table = getConfigKeyTable(); + expect(table.map((r) => r.key)).toEqual([...CONFIG_KEY_NAMES]); + for (const row of table) expect(row.description).not.toBe(""); + + expect(table.find((r) => r.key === "llm.provider")).toMatchObject({ + type: "auto | openai | anthropic | ollama | passthrough", + default: "auto", + env: ["LIBSCOPE_LLM_PROVIDER"], + secret: false, + }); + expect(table.find((r) => r.key === "indexing.allowSelfSignedCerts")).toMatchObject({ + type: "boolean", + default: "false", + env: ["LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS"], + }); + expect(table.find((r) => r.key === "indexing.maxDocumentSize")).toMatchObject({ + type: "integer", + default: String(100 * 1024 * 1024), + }); + expect(table.find((r) => r.key === "mcp.toolsets")).toMatchObject({ + type: "list of core | admin | all", + default: "none", + env: ["LIBSCOPE_MCP_TOOLSETS"], + }); + expect(table.find((r) => r.key === "openai.apiKey")).toMatchObject({ + env: ["LIBSCOPE_OPENAI_API_KEY", "OPENAI_API_KEY"], + secret: true, + }); + expect(table.find((r) => r.key === "anthropic.apiKey")).toMatchObject({ + env: ["LIBSCOPE_ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY"], + secret: true, + }); + }); +}); + +describe("setUserConfigValue", () => { + it.each([ + ["embedding.provider", "ollama", "ollama"], + ["embedding.model", "mxbai-embed-large", "mxbai-embed-large"], + ["embedding.url", "http://gpu:11434", "http://gpu:11434"], + ["embedding.dimensions", "1024", 1024], + ["llm.provider", "anthropic", "anthropic"], + ["llm.model", "gpt-4o", "gpt-4o"], + ["llm.url", "http://localhost:11434", "http://localhost:11434"], + ["database.path", "/data/libscope.db", "/data/libscope.db"], + ["logging.level", "debug", "debug"], + ["indexing.maxDocumentSize", "2048", 2048], + ["indexing.allowPrivateUrls", "true", true], + ["indexing.allowSelfSignedCerts", "0", false], + ] as const)("sets %s in config.json", (key, input, expected) => { + expect(setUserConfigValue(key, input)).toBe(expected); + const [section = "", field = ""] = key.split("."); + const sectionObj = readSavedRaw()[section] as Record<string, unknown>; + expect(sectionObj[field]).toBe(expected); + expect(existsSync(secretsPath())).toBe(false); }); - it("should not persist embedding.openaiApiKey to disk", () => { - saveUserConfig({ embedding: { provider: "openai", openaiApiKey: "sk-test-key" } }); + it("stores a list key as an array", () => { + expect(setUserConfigValue("mcp.toolsets", "admin, core")).toEqual(["admin", "core"]); + expect(readSavedRaw()["mcp"]).toEqual({ toolsets: ["admin", "core"] }); + expect(() => setUserConfigValue("mcp.toolsets", "admin,bogus")).toThrow(/mcp\.toolsets/); + }); - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); - const parsed = readSavedConfig(); + it("rejects unknown and old keys and lists valid keys", () => { + expect(() => setUserConfigValue("embedding.nope", "x")).toThrow( + /Unknown config key.*llm\.model/, + ); + expect(() => setUserConfigValue("embedding.ollamaModel", "x")).toThrow(/Unknown config key/); + expect(() => setUserConfigValue("embedding.openaiApiKey", "x")).toThrow(/Unknown config key/); + }); - expect(parsed.embedding.provider).toBe("openai"); - expect(parsed.embedding.openaiApiKey).toBeUndefined(); - expect(written).not.toContain("sk-test-key"); + it("rejects values outside an enum", () => { + expect(() => setUserConfigValue("embedding.provider", "cohere")).toThrow( + /one of: local, ollama, openai/, + ); + expect(() => setUserConfigValue("llm.provider", "gemini")).toThrow(/one of: auto/); + expect(() => setUserConfigValue("logging.level", "loud")).toThrow(/one of/); }); - it("should not persist llm.openaiApiKey to disk", () => { - saveUserConfig({ llm: { provider: "openai", openaiApiKey: "sk-llm-key" } }); + it("rejects non-boolean and non-numeric values", () => { + expect(() => setUserConfigValue("indexing.allowPrivateUrls", "yes")).toThrow(/true or false/); + expect(() => setUserConfigValue("indexing.maxDocumentSize", "-5")).toThrow(/positive integer/); + expect(() => setUserConfigValue("indexing.maxDocumentSize", "1.5")).toThrow(/positive integer/); + expect(() => setUserConfigValue("embedding.dimensions", "20000")).toThrow(/dimensions/); + expect(() => setUserConfigValue("llm.model", " ")).toThrow(/must not be empty/); + }); - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); - const parsed = readSavedConfig(); + it("writes the file with mode 0600, also when it already existed", () => { + writeExisting({}); + chmodSync(configPath(), 0o644); + setUserConfigValue("logging.level", "info"); + expect(statSync(configPath()).mode & 0o777).toBe(0o600); + }); - expect(parsed.llm?.openaiApiKey).toBeUndefined(); - expect(written).not.toContain("sk-llm-key"); + it("keeps registries, unknown top-level keys, and sibling keys", () => { + writeExisting({ + registries: [REGISTRY], + customTool: { a: 1 }, + embedding: { url: "http://gpu:11434" }, + }); + setUserConfigValue("embedding.provider", "ollama"); + const raw = readSavedRaw(); + expect(raw["registries"]).toEqual([REGISTRY]); + expect(raw["customTool"]).toEqual({ a: 1 }); + expect(raw["embedding"]).toEqual({ url: "http://gpu:11434", provider: "ollama" }); }); - it("should not persist llm.anthropicApiKey to disk", () => { - saveUserConfig({ llm: { provider: "anthropic", anthropicApiKey: "sk-ant-key" } }); + it("invalidates the config cache so the next load sees the change", () => { + loadConfig(); + setUserConfigValue("logging.level", "error"); + expect(loadConfig().logging.level).toBe("error"); + }); - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); - const parsed = readSavedConfig(); + it("keeps registries written by the registry module", () => { + saveRegistries([REGISTRY]); + setUserConfigValue("embedding.provider", "ollama"); + expect(loadRegistries()).toEqual([REGISTRY]); + expect(loadConfig().embedding.provider).toBe("ollama"); + }); - expect(parsed.llm?.anthropicApiKey).toBeUndefined(); - expect(written).not.toContain("sk-ant-key"); + it("registry writes keep values written by config set", () => { + setUserConfigValue("llm.model", "gpt-4o"); + saveRegistries([REGISTRY]); + expect(readSavedRaw()["llm"]).toEqual({ model: "gpt-4o" }); + expect(statSync(configPath()).mode & 0o777).toBe(0o600); }); +}); + +describe("secrets", () => { + it("API keys are written to secrets.json (0600), never to config.json", () => { + writeExisting({ logging: { level: "warn" } }); + expect(setUserConfigValue("openai.apiKey", "sk-openai-test")).toBe("sk-openai-test"); + expect(setUserConfigValue("anthropic.apiKey", "sk-ant-test")).toBe("sk-ant-test"); - it("should strip all credential fields simultaneously", () => { - saveUserConfig({ - embedding: { provider: "openai", openaiApiKey: "sk-embed" }, - llm: { provider: "openai", openaiApiKey: "sk-llm", anthropicApiKey: "sk-ant" }, + expect(readJson(secretsPath())).toEqual({ + openai: { apiKey: "sk-openai-test" }, + anthropic: { apiKey: "sk-ant-test" }, }); + expect(statSync(secretsPath()).mode & 0o777).toBe(0o600); + expect(readFileSync(configPath(), "utf-8")).not.toContain("sk-"); + expect(readSavedRaw()).toEqual({ logging: { level: "warn" } }); + }); - const written = readFileSync(join(tempHome, ".libscope", "config.json"), "utf-8"); + it("tightens an existing secrets.json to 0600", () => { + writeExisting({}, secretsPath()); + chmodSync(secretsPath(), 0o644); + setUserConfigValue("openai.apiKey", "sk-x"); + expect(statSync(secretsPath()).mode & 0o777).toBe(0o600); + }); - expect(written).not.toContain("sk-embed"); - expect(written).not.toContain("sk-llm"); - expect(written).not.toContain("sk-ant"); - expect(written).not.toContain("ApiKey"); + it("keys written by config set are loaded and masked by config get", () => { + setUserConfigValue("openai.apiKey", "sk-proj-abcdefghijkl9876"); + const config = loadConfig(); + expect(config.openai?.apiKey).toBe("sk-proj-abcdefghijkl9876"); + expect(getConfigValue(config, "openai.apiKey")).toBe("sk-…9876"); }); - it("should preserve non-credential config fields", () => { - saveUserConfig({ - embedding: { - provider: "openai", - openaiModel: "text-embedding-3-large", - openaiApiKey: "sk-test", - }, - logging: { level: "debug" }, - }); + it("getConfigFileFor names the file a key is stored in", () => { + expect(getConfigFileFor("openai.apiKey")).toBe(getSecretsPath()); + expect(getConfigFileFor("llm.model")).toBe(getUserConfigPath()); + expect(getSecretsPath()).toBe(secretsPath()); + }); +}); + +describe("unsetUserConfigValue", () => { + it("removes the key and drops an empty section", () => { + writeExisting({ logging: { level: "debug" }, registries: [REGISTRY] }); + expect(unsetUserConfigValue("logging.level")).toBe(true); + expect(readSavedRaw()).toEqual({ registries: [REGISTRY] }); + }); + + it("keeps other keys in the section", () => { + writeExisting({ embedding: { provider: "ollama", model: "m" } }); + unsetUserConfigValue("embedding.model"); + expect(readSavedRaw()["embedding"]).toEqual({ provider: "ollama" }); + }); + + it("returns false when the key is not set", () => { + expect(unsetUserConfigValue("llm.model")).toBe(false); + }); + + it("removes an API key from secrets.json", () => { + setUserConfigValue("openai.apiKey", "sk-old"); + setUserConfigValue("anthropic.apiKey", "sk-ant-keep"); + expect(unsetUserConfigValue("openai.apiKey")).toBe(true); + expect(readJson(secretsPath())).toEqual({ anthropic: { apiKey: "sk-ant-keep" } }); + }); - const parsed = readSavedConfig(); + it("rejects unknown keys", () => { + expect(() => unsetUserConfigValue("bogus")).toThrow(/Unknown config key/); + }); +}); + +describe("getConfigValue", () => { + it("returns effective values including defaults", () => { + setUserConfigValue("indexing.maxDocumentSize", "4096"); + const config = loadConfig(); + expect(getConfigValue(config, "indexing.maxDocumentSize")).toBe(4096); + expect(getConfigValue(config, "logging.level")).toBe("info"); + expect(getConfigValue(config, "llm.provider")).toBe("auto"); + }); + + it("returns undefined for unset optional keys", () => { + expect(getConfigValue(loadConfig(), "llm.model")).toBeUndefined(); + expect(getConfigValue(loadConfig(), "openai.apiKey")).toBeUndefined(); + }); - expect(parsed.embedding.provider).toBe("openai"); - expect(parsed.embedding.openaiModel).toBe("text-embedding-3-large"); - expect(parsed.logging.level).toBe("debug"); - expect(parsed.embedding.openaiApiKey).toBeUndefined(); + it("reports the user config path under HOME", () => { + expect(getUserConfigPath()).toBe(configPath()); }); }); diff --git a/tests/unit/config.test.ts b/tests/unit/config.test.ts index fa34f2e6..44a56243 100644 --- a/tests/unit/config.test.ts +++ b/tests/unit/config.test.ts @@ -21,7 +21,7 @@ describe("config", () => { expect(config.embedding.provider).toBe("local"); expect(config.logging.level).toBe("info"); - expect(config.database.path).toContain("libscope.db"); + // database.path defaults are covered with a temp HOME in db-location.test.ts }); it("should return cached config on repeated calls", () => { @@ -43,21 +43,21 @@ describe("config", () => { it("should pick up LIBSCOPE_OPENAI_API_KEY", () => { const config = loadConfigWithEnv({ LIBSCOPE_OPENAI_API_KEY: "sk-test123" }); - expect(config.embedding.openaiApiKey).toBe("sk-test123"); + expect(config.openai?.apiKey).toBe("sk-test123"); }); - it("should pick up LIBSCOPE_OLLAMA_URL", () => { - const config = loadConfigWithEnv({ LIBSCOPE_OLLAMA_URL: "http://custom:11434" }); - expect(config.embedding.ollamaUrl).toBe("http://custom:11434"); + it("should pick up LIBSCOPE_EMBEDDING_URL", () => { + const config = loadConfigWithEnv({ LIBSCOPE_EMBEDDING_URL: "http://custom:11434" }); + expect(config.embedding.url).toBe("http://custom:11434"); }); - it("should pick up LIBSCOPE_ALLOW_PRIVATE_URLS", () => { - const config = loadConfigWithEnv({ LIBSCOPE_ALLOW_PRIVATE_URLS: "true" }); + it("should pick up LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS", () => { + const config = loadConfigWithEnv({ LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS: "true" }); expect(config.indexing.allowPrivateUrls).toBe(true); }); - it("should pick up LIBSCOPE_ALLOW_SELF_SIGNED_CERTS", () => { - const config = loadConfigWithEnv({ LIBSCOPE_ALLOW_SELF_SIGNED_CERTS: "1" }); + it("should pick up LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS", () => { + const config = loadConfigWithEnv({ LIBSCOPE_INDEXING_ALLOW_SELF_SIGNED_CERTS: "1" }); expect(config.indexing.allowSelfSignedCerts).toBe(true); }); @@ -75,15 +75,14 @@ function makeConfig(overrides: Partial<LibScopeConfig> = {}): LibScopeConfig { return { embedding: { provider: "local", - ollamaUrl: "http://localhost:11434", - ollamaModel: "nomic-embed-text", - openaiModel: "text-embedding-3-small", ...overrides.embedding, }, database: { path: "/tmp/test-libscope/libscope.db", ...overrides.database }, indexing: { maxDocumentSize: 100 * 1024 * 1024, ...overrides.indexing }, logging: { level: "info", ...overrides.logging }, ...("llm" in overrides ? { llm: overrides.llm } : {}), + ...("openai" in overrides ? { openai: overrides.openai } : {}), + ...("anthropic" in overrides ? { anthropic: overrides.anthropic } : {}), }; } @@ -138,17 +137,32 @@ describe("validateConfig", () => { it("should not warn when embedding provider is openai with config key", () => { const config = makeConfig({ - embedding: { provider: "openai", openaiApiKey: "sk-test" }, + embedding: { provider: "openai" }, + openai: { apiKey: "sk-test" }, }); const warnings = validateConfig(config); expect(warnings).toHaveLength(0); }); - it("should not warn when embedding provider is openai with OPENAI_API_KEY env", () => { + it("should validate the resolved config only (env keys are resolved by loadConfig)", () => { process.env["OPENAI_API_KEY"] = "sk-env-test"; const config = makeConfig({ embedding: { provider: "openai" } }); const warnings = validateConfig(config); - expect(warnings).toHaveLength(0); + expect(warnings[0]).toContain("LIBSCOPE_OPENAI_API_KEY or OPENAI_API_KEY"); + }); + + it("should warn when llm provider is anthropic without API key", () => { + const config = makeConfig({ llm: { provider: "anthropic" } }); + const warnings = validateConfig(config); + expect(warnings.some((w) => w.includes("LIBSCOPE_ANTHROPIC_API_KEY"))).toBe(true); + }); + + it("should not warn when llm provider is anthropic with a key", () => { + const config = makeConfig({ + llm: { provider: "anthropic" }, + anthropic: { apiKey: "sk-ant" }, + }); + expect(validateConfig(config)).toHaveLength(0); }); it("should warn when llm provider is openai without API key", () => { @@ -159,26 +173,29 @@ describe("validateConfig", () => { expect(warnings.some((w) => w.includes("llm.provider"))).toBe(true); }); - it("should not warn when llm provider is openai with embedding key available", () => { + it("should not warn when llm provider is openai with an OpenAI key", () => { const config = makeConfig({ - embedding: { provider: "local", openaiApiKey: "sk-shared" }, llm: { provider: "openai" }, + openai: { apiKey: "sk-shared" }, }); const warnings = validateConfig(config); expect(warnings.some((w) => w.includes("llm.provider"))).toBe(false); }); - it("should warn when ollama provider has no URL", () => { - const config = makeConfig({ - embedding: { provider: "ollama", ollamaUrl: undefined }, - }); + it("should not warn when ollama provider has no URL (the default URL applies)", () => { + const config = makeConfig({ embedding: { provider: "ollama" } }); + expect(validateConfig(config)).toHaveLength(0); + }); + + it("should warn when embedding.model is set for the local provider", () => { + const config = makeConfig({ embedding: { provider: "local", model: "bge-m3" } }); const warnings = validateConfig(config); - expect(warnings.some((w) => w.includes("ollamaUrl"))).toBe(true); + expect(warnings.some((w) => w.includes("ignored by the local provider"))).toBe(true); }); it("should not warn when ollama provider has a URL", () => { const config = makeConfig({ - embedding: { provider: "ollama", ollamaUrl: "http://localhost:11434" }, + embedding: { provider: "ollama", url: "http://localhost:11434" }, }); const warnings = validateConfig(config); expect(warnings).toHaveLength(0); diff --git a/tests/unit/confirm.test.ts b/tests/unit/confirm.test.ts index 1a81b025..43121c7a 100644 --- a/tests/unit/confirm.test.ts +++ b/tests/unit/confirm.test.ts @@ -1,5 +1,5 @@ -import { describe, it, expect, vi } from "vitest"; -import { confirmAction } from "../../src/cli/confirm.js"; +import { afterEach, describe, it, expect, vi } from "vitest"; +import { confirmAction, confirmOrCancel } from "../../src/cli/confirm.js"; import { EventEmitter } from "node:events"; function createMockInterface(answer: string) { @@ -18,40 +18,46 @@ describe("confirmAction", () => { expect(result).toBe(true); }); - it("returns true when user answers 'y'", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("y")); - expect(result).toBe(true); - }); - - it("returns true when user answers 'yes'", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("yes")); - expect(result).toBe(true); + it.each([ + { answer: "y", expected: true }, + { answer: "yes", expected: true }, + { answer: "YES", expected: true }, + { answer: "n", expected: false }, + { answer: "", expected: false }, + { answer: "maybe", expected: false }, + ])("returns $expected when user answers '$answer'", async ({ answer, expected }) => { + const result = await confirmAction("Delete?", false, createMockInterface(answer)); + expect(result).toBe(expected); }); - it("returns true when user answers 'YES' (case-insensitive)", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("YES")); - expect(result).toBe(true); + it("closes the readline interface after prompting", async () => { + const factory = createMockInterface("y"); + const rl = factory(); + await confirmAction("Delete?", false, () => rl); + expect(rl.close).toHaveBeenCalled(); }); +}); - it("returns false when user answers 'n'", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("n")); - expect(result).toBe(false); +describe("confirmOrCancel", () => { + afterEach(() => { + vi.restoreAllMocks(); }); - it("returns false when user answers empty string", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("")); - expect(result).toBe(false); + it("returns true without printing when yes flag is set", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + expect(await confirmOrCancel("Delete?", true)).toBe(true); + expect(log).not.toHaveBeenCalled(); }); - it("returns false when user answers arbitrary text", async () => { - const result = await confirmAction("Delete?", false, createMockInterface("maybe")); - expect(result).toBe(false); + it("returns true without printing when user confirms", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + expect(await confirmOrCancel("Delete?", undefined, createMockInterface("y"))).toBe(true); + expect(log).not.toHaveBeenCalled(); }); - it("closes the readline interface after prompting", async () => { - const factory = createMockInterface("y"); - const rl = factory(); - await confirmAction("Delete?", false, () => rl); - expect(rl.close).toHaveBeenCalled(); + it("prints 'Cancelled.' and returns false when user declines", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + expect(await confirmOrCancel("Delete?", false, createMockInterface("n"))).toBe(false); + expect(log).toHaveBeenCalledWith("Cancelled."); }); }); diff --git a/tests/unit/connector-abort.test.ts b/tests/unit/connector-abort.test.ts new file mode 100644 index 00000000..0f3a44d5 --- /dev/null +++ b/tests/unit/connector-abort.test.ts @@ -0,0 +1,290 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdirSync, writeFileSync, rmSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { ValidationError } from "../../src/errors.js"; + +// Connector state and saved configs live under a temp HOME — never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-abort-init-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const { syncNotion } = await import("../../src/connectors/notion.js"); +const { syncSlack, _setRateLimitDelay } = await import("../../src/connectors/slack.js"); +const { syncConfluence } = await import("../../src/connectors/confluence.js"); +const { syncObsidianVault } = await import("../../src/connectors/obsidian.js"); +const { syncOneNote, _resetRateLimiter } = await import("../../src/connectors/onenote.js"); +const { resolveSyncConfig, saveConnectorSettings, saveRefreshedOneNoteTokens } = + await import("../../src/connectors/saved-config.js"); +const { getConnectorsDir } = await import("../../src/connectors/index.js"); + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +function onlySyncRow(db: Database.Database): { status: string; connector_name: string } { + const rows = db.prepare("SELECT status, connector_name FROM connector_syncs").all() as Array<{ + status: string; + connector_name: string; + }>; + expect(rows).toHaveLength(1); + return rows[0]!; +} + +describe("connector abort signals", () => { + let db: Database.Database; + const provider = new MockEmbeddingProvider(); + + beforeEach(() => { + initLogger("silent"); + tempHome = join(tmpdir(), `libscope-abort-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + db = createTestDbWithVec(); + _setRateLimitDelay(0); + _resetRateLimiter(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); + + it("notion stops between items and records the run as failed", async () => { + const controller = new AbortController(); + const fetchMock = vi.fn((url: string) => { + if (url.endsWith("/v1/search")) { + return Promise.resolve( + json({ + results: ["p1", "p2"].map((id) => ({ + object: "page", + id, + last_edited_time: "2026-01-01T00:00:00.000Z", + properties: { Name: { type: "title", title: [{ plain_text: id }] } }, + })), + has_more: false, + next_cursor: null, + }), + ); + } + controller.abort(); // cancelled while the first page is being fetched + return Promise.resolve( + json({ + results: [ + { id: "b", type: "paragraph", paragraph: { rich_text: [{ plain_text: "x" }] } }, + ], + has_more: false, + next_cursor: null, + }), + ); + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + syncNotion(db, provider, { token: "secret_t" }, { signal: controller.signal, syncName: "n" }), + ).rejects.toThrow(); + + const urls = fetchMock.mock.calls.map((c) => c[0]); + expect(urls.some((u) => u.includes("/blocks/p1/"))).toBe(true); + expect(urls.some((u) => u.includes("/blocks/p2/"))).toBe(false); + expect(onlySyncRow(db)).toEqual({ status: "failed", connector_name: "n" }); + }); + + it("slack stops before the next channel", async () => { + const controller = new AbortController(); + const fetchMock = vi.fn((url: string) => { + if (url.includes("conversations.list")) { + return Promise.resolve( + json({ + ok: true, + channels: [ + { id: "C1", name: "one" }, + { id: "C2", name: "two" }, + ], + }), + ); + } + controller.abort(); + return Promise.resolve(json({ ok: true, messages: [] })); + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + syncSlack( + db, + provider, + { token: "xoxb", channels: ["all"], threadMode: "aggregate" }, + { signal: controller.signal }, + ), + ).rejects.toThrow(); + + const historyCalls = fetchMock.mock.calls.filter((c) => c[0].includes("conversations.history")); + expect(historyCalls).toHaveLength(1); + expect(onlySyncRow(db).status).toBe("failed"); + }); + + it("confluence does not call the API when already aborted", async () => { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + const controller = new AbortController(); + controller.abort(); + + await expect( + syncConfluence( + db, + provider, + { baseUrl: "https://acme.atlassian.net", email: "e", token: "t", spaces: ["all"] }, + { signal: controller.signal }, + ), + ).rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + expect(onlySyncRow(db).status).toBe("failed"); + }); + + it("obsidian indexes nothing and keeps no vault state when aborted", async () => { + const vault = join(tempHome, "vault"); + mkdirSync(vault); + writeFileSync(join(vault, "a.md"), "# A\n\nAlpha note."); + writeFileSync(join(vault, "b.md"), "# B\n\nBeta note."); + const controller = new AbortController(); + controller.abort(); + + await expect( + syncObsidianVault( + db, + provider, + { vaultPath: vault, topicMapping: "folder", excludePatterns: [] }, + { signal: controller.signal }, + ), + ).rejects.toThrow(); + expect(db.prepare("SELECT COUNT(*) AS n FROM documents").get()).toEqual({ n: 0 }); + expect(onlySyncRow(db).status).toBe("failed"); + }); + + it("onenote does not delete unseen pages when aborted mid-sync", async () => { + db.prepare( + "INSERT INTO documents (id, source_type, title, content, url) VALUES ('keep', 'topic', 'K', 'Body', 'onenote://NB/S/K')", + ).run(); + const controller = new AbortController(); + vi.stubGlobal( + "fetch", + vi.fn(() => { + controller.abort(); + return Promise.resolve(json({ value: [{ id: "nb1", displayName: "NB" }] })); + }), + ); + + await expect( + syncOneNote( + db, + provider, + { + clientId: "", + tenantId: "common", + accessToken: "a", + tokenExpiry: "2999-01-01T00:00:00.000Z", + notebooks: ["all"], + excludeSections: [], + }, + { signal: controller.signal }, + ), + ).rejects.toThrow(); + expect(db.prepare("SELECT id FROM documents").all()).toEqual([{ id: "keep" }]); + expect(onlySyncRow(db).status).toBe("failed"); + }); +}); + +describe("resolveSyncConfig (MCP sync tools)", () => { + beforeEach(() => { + initLogger("silent"); + tempHome = join(tmpdir(), `libscope-resolve-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + }); + + afterEach(() => { + rmSync(tempHome, { recursive: true, force: true }); + }); + + it("fills omitted parameters from the saved config, then defaults", () => { + saveConnectorSettings("slack", "slack", { token: "xoxb-saved", channels: ["eng"] }); + const { config, saved } = resolveSyncConfig<{ + token: string; + channels: string[]; + threadMode: string; + }>( + "slack", + "slack", + { token: undefined, channels: undefined, threadMode: undefined }, + { defaults: { channels: ["all"], threadMode: "aggregate" }, required: ["token"] }, + ); + expect(config).toMatchObject({ + token: "xoxb-saved", + channels: ["eng"], + threadMode: "aggregate", + }); + expect(saved).toBeDefined(); + }); + + it("lets explicit parameters override the saved config", () => { + saveConnectorSettings("notion", "team", { token: "secret_saved" }); + const { config } = resolveSyncConfig<{ token: string }>("notion", "team", { + token: "secret_given", + }); + expect(config.token).toBe("secret_given"); + }); + + it("names the missing fields and the config name when nothing is saved", () => { + expect(() => + resolveSyncConfig<{ baseUrl: string; token: string }>( + "confluence", + "wiki", + { baseUrl: undefined, token: undefined }, + { required: ["baseUrl", "token"] }, + ), + ).toThrow(ValidationError); + expect(() => + resolveSyncConfig<{ token: string }>("notion", "absent", {}, { required: ["token"] }), + ).toThrow(/token.*"absent"/); + }); + + it("saves refreshed OneNote tokens only when the connector refreshed them", () => { + const saved = { + clientId: "c", + tenantId: "common", + accessToken: "old", + refreshToken: "r1", + notebooks: ["all"], + excludeSections: [], + }; + saveConnectorSettings("onenote", "onenote", saved); + const path = join(getConnectorsDir(), "onenote.json"); + + saveRefreshedOneNoteTokens("onenote", saved, { + ...saved, + accessToken: "x", + refreshToken: undefined, + }); + expect(JSON.parse(readFileSync(path, "utf-8"))).toMatchObject({ accessToken: "old" }); + + saveRefreshedOneNoteTokens("onenote", saved, { + ...saved, + accessToken: "new", + refreshToken: "r2", + tokenExpiry: "2999-01-01T00:00:00.000Z", + }); + expect(JSON.parse(readFileSync(path, "utf-8"))).toMatchObject({ + accessToken: "new", + refreshToken: "r2", + }); + }); +}); diff --git a/tests/unit/connector-registry.test.ts b/tests/unit/connector-registry.test.ts new file mode 100644 index 00000000..986de824 --- /dev/null +++ b/tests/unit/connector-registry.test.ts @@ -0,0 +1,129 @@ +import { describe, it, expect, vi, afterAll } from "vitest"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + CONNECTORS, + getConnector, + maskConnectorSecrets, + saveConnection, +} from "../../src/connectors/registry.js"; +import { CONNECTOR_TYPES } from "../../src/connectors/saved-config.js"; +import { createTestDb } from "../fixtures/test-db.js"; +import { ValidationError } from "../../src/errors.js"; + +// Saved connections live under a temp HOME, never the real ~/.libscope. +const tempHome = mkdtempSync(join(tmpdir(), "libscope-registry-")); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +afterAll(() => { + rmSync(tempHome, { recursive: true, force: true }); +}); + +describe("connector registry", () => { + it("has one definition per connector type", () => { + expect(Object.keys(CONNECTORS).sort()).toEqual([...CONNECTOR_TYPES].sort()); + for (const type of CONNECTOR_TYPES) { + expect(getConnector(type).type).toBe(type); + for (const field of getConnector(type).secretFields) { + expect(getConnector(type).configSchema.shape).toHaveProperty(field); + } + } + }); + + it("rejects unknown connector types", () => { + expect(() => getConnector("myspace")).toThrow(ValidationError); + }); + + it("masks secret fields and drops bookkeeping fields", () => { + expect( + maskConnectorSecrets("confluence", { + baseUrl: "https://acme.atlassian.net", + token: "secret", + connectorType: "confluence", + lastSync: "2026-01-01", + schedule: { cronExpression: "0 * * * *" }, + }), + ).toEqual({ baseUrl: "https://acme.atlassian.net", token: "***" }); + expect(maskConnectorSecrets("unknown", { token: "visible" })).toEqual({ token: "visible" }); + }); + + it("maps each connector's result to the same summary shape", () => { + expect( + CONNECTORS.onenote.summarize({ + notebooks: 1, + sections: 2, + pagesAdded: 3, + pagesUpdated: 4, + pagesDeleted: 5, + errors: [{ page: "p", error: "e" }], + }), + ).toEqual({ added: 3, updated: 4, deleted: 5, errors: ["p: e"] }); + expect( + CONNECTORS.slack.summarize({ + channels: 1, + messagesIndexed: 2, + threadsIndexed: 3, + errors: [{ channel: "c", error: "e" }], + }), + ).toEqual({ added: 5, updated: 0, deleted: 0, errors: ["#c: e"] }); + }); + + it("needs the saved path to disconnect a vault or docs site", () => { + const db = createTestDb(); + try { + expect(() => CONNECTORS.obsidian.disconnect(db, undefined)).toThrow(ValidationError); + expect(() => CONNECTORS.docs.disconnect(db, {})).toThrow(ValidationError); + expect(CONNECTORS.docs.disconnect(db, { url: "https://docs.example.com/" })).toBe(0); + } finally { + db.close(); + } + }); +}); + +describe("saveConnection", () => { + const saved = (name: string): Record<string, unknown> => + JSON.parse( + readFileSync(join(tempHome, ".libscope", "connectors", `${name}.json`), "utf-8"), + ) as Record<string, unknown>; + + it("validates against the type's schema and tags the connector type", () => { + expect(() => saveConnection("n1", "notion", {})).toThrow(/Invalid Notion settings: token/); + saveConnection("n1", "notion", { token: "secret_x", excludePages: undefined }); + expect(saved("n1")).toEqual({ token: "secret_x", connectorType: "notion" }); + }); + + it("merges over the saved settings and sets, keeps or removes the schedule", () => { + saveConnection( + "s1", + "slack", + { token: "a", channels: ["all"], threadMode: "aggregate" }, + { + schedule: "0 * * * *", + }, + ); + saveConnection("s1", "slack", { token: "b" }); + expect(saved("s1")).toMatchObject({ + token: "b", + channels: ["all"], + schedule: { cronExpression: "0 * * * *" }, + }); + saveConnection("s1", "slack", {}, { schedule: null }); + expect(saved("s1")["schedule"]).toBeUndefined(); + expect(() => saveConnection("s1", "slack", {}, { schedule: "every hour" })).toThrow( + ValidationError, + ); + }); + + it("refuses to change a saved connection into another type", () => { + saveConnection("v1", "obsidian", { + vaultPath: "/v", + topicMapping: "folder", + excludePatterns: [], + }); + expect(() => saveConnection("v1", "notion", { token: "t" })).toThrow(/obsidian connector/); + }); +}); diff --git a/tests/unit/connector-saved-config.test.ts b/tests/unit/connector-saved-config.test.ts new file mode 100644 index 00000000..2cecd48d --- /dev/null +++ b/tests/unit/connector-saved-config.test.ts @@ -0,0 +1,303 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdirSync, writeFileSync, readFileSync, rmSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { ConfigError } from "../../src/errors.js"; + +// Every config file lives under a temp HOME — never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-saved-init-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal<typeof import("node:os")>(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const cronCallbacks = new Map<string, () => void>(); +vi.mock("node-cron", () => ({ + default: { + validate: (): boolean => true, + schedule: vi.fn((expr: string, callback: () => void) => { + cronCallbacks.set(expr, callback); + return { stop: vi.fn() }; + }), + }, +})); + +const { + saveConnectorSettings, + loadSavedConnectorConfig, + findSavedOneNoteConfig, + resolveConnectorType, +} = await import("../../src/connectors/saved-config.js"); +const { runSavedConnectorSync } = await import("../../src/connectors/registry.js"); +const { saveConnectorConfig, getConnectorsDir } = await import("../../src/connectors/index.js"); +const { ConnectorScheduler, loadScheduleEntries } = await import("../../src/core/scheduler.js"); +const { _resetRateLimiter } = await import("../../src/connectors/onenote.js"); + +interface SyncRow { + connector_type: string; + connector_name: string; + status: string; + docs_added: number; + docs_updated: number; + docs_deleted: number; + error_message: string | null; +} + +function syncRows(db: Database.Database): SyncRow[] { + return db.prepare("SELECT * FROM connector_syncs ORDER BY id").all() as SyncRow[]; +} + +function readSaved(name: string): Record<string, unknown> { + return JSON.parse(readFileSync(join(getConnectorsDir(), `${name}.json`), "utf-8")) as Record< + string, + unknown + >; +} + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +/** Notion API: one page with one paragraph. */ +function mockNotionApi(): ReturnType<typeof vi.fn> { + const fetchMock = vi.fn((url: string) => { + if (url.endsWith("/v1/search")) { + return Promise.resolve( + json({ + results: [ + { + object: "page", + id: "page-1", + last_edited_time: "2026-01-01T00:00:00.000Z", + properties: { Name: { type: "title", title: [{ plain_text: "Runbook" }] } }, + }, + ], + has_more: false, + next_cursor: null, + }), + ); + } + return Promise.resolve( + json({ + results: [ + { + id: "b1", + type: "paragraph", + paragraph: { rich_text: [{ plain_text: "Restart the service." }] }, + }, + ], + has_more: false, + next_cursor: null, + }), + ); + }); + vi.stubGlobal("fetch", fetchMock); + return fetchMock; +} + +describe("saved connector configs", () => { + let db: Database.Database; + const provider = new MockEmbeddingProvider(); + + beforeEach(() => { + initLogger("silent"); + tempHome = join(tmpdir(), `libscope-saved-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + db = createTestDbWithVec(); + cronCallbacks.clear(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); + + describe("saveConnectorSettings", () => { + it("tags the connector type, writes mode 0600 and keeps an existing schedule", () => { + saveConnectorSettings("notion", "team", { token: "secret_a" }); + const path = join(getConnectorsDir(), "team.json"); + expect(statSync(path).mode & 0o777).toBe(0o600); + + const withSchedule = { ...readSaved("team"), schedule: { cronExpression: "0 * * * *" } }; + writeFileSync(path, JSON.stringify(withSchedule)); + saveConnectorSettings("notion", "team", { token: "secret_b" }); + + expect(readSaved("team")).toEqual({ + token: "secret_b", + connectorType: "notion", + schedule: { cronExpression: "0 * * * *" }, + }); + }); + }); + + describe("resolveConnectorType / loadSavedConnectorConfig", () => { + it("prefers connectorType over Confluence's own cloud/server type field", () => { + expect(resolveConnectorType("x", { connectorType: "confluence", type: "cloud" })).toBe( + "confluence", + ); + expect(resolveConnectorType("x", { type: "notion" })).toBe("notion"); + expect(resolveConnectorType("slack", { token: "t" })).toBe("slack"); + }); + + it("rejects a config that declares a different connector type", () => { + saveConnectorSettings("slack", "work", { token: "xoxb", channels: ["all"] }); + expect(() => loadSavedConnectorConfig("notion", "work")).toThrow(ConfigError); + expect(loadSavedConnectorConfig("slack", "work")).toMatchObject({ token: "xoxb" }); + }); + + it("schedules a saved Confluence config as confluence, not as its cloud type", () => { + saveConnectorSettings("confluence", "wiki", { + baseUrl: "https://acme.atlassian.net", + type: "cloud", + token: "t", + spaces: ["all"], + }); + const path = join(getConnectorsDir(), "wiki.json"); + writeFileSync( + path, + JSON.stringify({ ...readSaved("wiki"), schedule: { cronExpression: "0 0 * * *" } }), + ); + expect(loadScheduleEntries()).toEqual([ + { connectorType: "confluence", connectorName: "wiki", cronExpression: "0 0 * * *" }, + ]); + }); + }); + + describe("runSavedConnectorSync", () => { + it("writes exactly one sync row under the configured name and saves lastSync", async () => { + mockNotionApi(); + saveConnectorSettings("notion", "team-notion", { token: "secret_abc" }); + + const before = new Date().toISOString(); + await runSavedConnectorSync(db, provider, "notion", "team-notion"); + + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + connector_type: "notion", + connector_name: "team-notion", + status: "completed", + docs_added: 1, + }); + const saved = readSaved("team-notion"); + expect(saved["connectorType"]).toBe("notion"); + expect(String(saved["lastSync"]) >= before).toBe(true); + }); + + it("records one failed row when the connector rejects the saved config", async () => { + saveConnectorSettings("notion", "bad", { token: "not-a-notion-token" }); + await expect(runSavedConnectorSync(db, provider, "notion", "bad")).rejects.toThrow( + "Notion token must start with", + ); + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ connector_name: "bad", status: "failed" }); + expect(readSaved("bad")["lastSync"]).toBeUndefined(); + }); + + it("records one failed row when the saved config is missing", async () => { + await expect(runSavedConnectorSync(db, provider, "slack", "nope")).rejects.toThrow( + ConfigError, + ); + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + connector_type: "slack", + connector_name: "nope", + status: "failed", + }); + }); + + it("refreshes an expired OneNote token, saves the new tokens and maps deletions", async () => { + _resetRateLimiter(); + db.prepare( + "INSERT INTO documents (id, source_type, title, content, url) VALUES ('old', 'topic', 'Old', 'Body', 'onenote://NB/Sec/Old')", + ).run(); + saveConnectorSettings("onenote", "onenote", { + clientId: "client-1", + tenantId: "common", + accessToken: "expired-access", + refreshToken: "refresh-1", + tokenExpiry: "2000-01-01T00:00:00.000Z", + notebooks: ["all"], + excludeSections: [], + }); + + const fetchMock = vi.fn((url: string, init?: RequestInit) => { + if (url.includes("/oauth2/v2.0/token")) { + expect((init?.body as URLSearchParams).get("refresh_token")).toBe("refresh-1"); + return Promise.resolve( + json({ access_token: "new-access", refresh_token: "refresh-2", expires_in: 3600 }), + ); + } + const auth = (init?.headers as Record<string, string>)["Authorization"]; + expect(auth).toBe("Bearer new-access"); + return Promise.resolve(json({ value: [] })); + }); + vi.stubGlobal("fetch", fetchMock); + + await runSavedConnectorSync(db, provider, "onenote", "onenote"); + + const saved = readSaved("onenote"); + expect(saved["accessToken"]).toBe("new-access"); + expect(saved["refreshToken"]).toBe("refresh-2"); + expect(Date.parse(String(saved["tokenExpiry"]))).toBeGreaterThan(Date.now()); + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ connector_name: "onenote", docs_deleted: 1 }); + }); + }); + + describe("findSavedOneNoteConfig", () => { + it("falls back to credentials stored by older versions in connectors.json", () => { + saveConnectorConfig({ + onenote: { clientId: "c", tenantId: "t", refreshToken: "r", lastSync: "2026-01-01" }, + }); + expect(findSavedOneNoteConfig()).toEqual({ + clientId: "c", + tenantId: "t", + accessToken: undefined, + refreshToken: "r", + tokenExpiry: undefined, + lastSync: "2026-01-01", + notebooks: ["all"], + excludeSections: [], + }); + expect(findSavedOneNoteConfig("other")).toBeUndefined(); + }); + }); + + describe("scheduler end to end", () => { + it("runs a scheduled Notion sync from a saved config and records one row", async () => { + mockNotionApi(); + saveConnectorSettings("notion", "notion", { token: "secret_abc" }); + const path = join(getConnectorsDir(), "notion.json"); + writeFileSync( + path, + JSON.stringify({ ...readSaved("notion"), schedule: { cronExpression: "*/5 * * * *" } }), + ); + + const scheduler = new ConnectorScheduler(db, provider); + scheduler.start(loadScheduleEntries()); + cronCallbacks.get("*/5 * * * *")!(); + await vi.waitFor(() => { + expect(scheduler.getStatus().jobs[0]!.lastRun).toBeDefined(); + }); + await scheduler.stop(); + + const rows = syncRows(db); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ connector_name: "notion", status: "completed" }); + expect(readSaved("notion")["schedule"]).toEqual({ cronExpression: "*/5 * * * *" }); + }); + }); +}); diff --git a/tests/unit/db-location.test.ts b/tests/unit/db-location.test.ts new file mode 100644 index 00000000..aa3fce60 --- /dev/null +++ b/tests/unit/db-location.test.ts @@ -0,0 +1,215 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + resolveDatabasePath, + resolveDbPath, + resetLegacyDatabaseWarning, + createDatabase, +} from "../../src/db/connection.js"; +import { runMigrations } from "../../src/db/schema.js"; +import { getStats } from "../../src/core/analytics.js"; +import { loadConfig, invalidateConfigCache, expandHomeDir } from "../../src/config.js"; +import { LibScope } from "../../src/LibScope.js"; +import { initLogger } from "../../src/logger.js"; +import * as loggerModule from "../../src/logger.js"; + +const ENV_KEYS = ["HOME", "LIBSCOPE_WORKSPACE", "LIBSCOPE_EMBEDDING_PROVIDER"]; + +describe("database location (temp HOME)", () => { + let tempHome: string; + let tempCwd: string; + const savedEnv: Record<string, string | undefined> = {}; + + const wsDb = (name: string): string => + join(tempHome, ".libscope", "workspaces", name, "libscope.db"); + const legacyDb = (): string => join(tempHome, ".libscope", "libscope.db"); + + function touch(path: string): void { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, ""); + } + + function writeUserConfig(data: unknown): void { + mkdirSync(join(tempHome, ".libscope"), { recursive: true }); + writeFileSync(join(tempHome, ".libscope", "config.json"), JSON.stringify(data)); + } + + beforeEach(() => { + initLogger("silent"); + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + tempHome = mkdtempSync(join(tmpdir(), "libscope-dbloc-home-")); + tempCwd = mkdtempSync(join(tmpdir(), "libscope-dbloc-cwd-")); + process.env["HOME"] = tempHome; + vi.spyOn(process, "cwd").mockReturnValue(tempCwd); + resetLegacyDatabaseWarning(); + invalidateConfigCache(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + for (const [key, val] of Object.entries(savedEnv)) { + if (val === undefined) delete process.env[key]; + else process.env[key] = val; + } + invalidateConfigCache(); + rmSync(tempHome, { recursive: true, force: true }); + rmSync(tempCwd, { recursive: true, force: true }); + }); + + describe("resolveDatabasePath", () => { + it("defaults to the active workspace database", () => { + expect(resolveDatabasePath()).toBe(wsDb("default")); + }); + + it("uses the workspace option", () => { + expect(resolveDatabasePath({ workspace: "work" })).toBe(wsDb("work")); + }); + + it("uses LIBSCOPE_WORKSPACE when no option is given", () => { + process.env["LIBSCOPE_WORKSPACE"] = "env-ws"; + expect(resolveDatabasePath()).toBe(wsDb("env-ws")); + }); + + it("explicit path wins over workspace", () => { + expect(resolveDatabasePath({ explicitPath: "/data/x.db", workspace: "work" })).toBe( + "/data/x.db", + ); + }); + + it("expands a leading ~ in an explicit path", () => { + expect(resolveDatabasePath({ explicitPath: "~/kb/libscope.db" })).toBe( + join(tempHome, "kb", "libscope.db"), + ); + expect(expandHomeDir("~")).toBe(tempHome); + expect(expandHomeDir("/abs/~/x")).toBe("/abs/~/x"); + }); + + it("resolveDbPath keeps its old contract", () => { + expect(resolveDbPath()).toBe(wsDb("default")); + expect(resolveDbPath("/tmp/explicit.db")).toBe("/tmp/explicit.db"); + }); + }); + + describe("legacy ~/.libscope/libscope.db notice", () => { + it("warns once, naming the legacy file and database.path, without touching it", () => { + touch(legacyDb()); + const warn = vi.fn(); + expect(resolveDatabasePath({ warn })).toBe(wsDb("default")); + resolveDatabasePath({ warn }); + expect(warn).toHaveBeenCalledTimes(1); + const message = String(warn.mock.calls[0]?.[0]); + expect(message).toContain(legacyDb()); + expect(message).toContain("libscope config set database.path"); + expect(existsSync(legacyDb())).toBe(true); + }); + + it("uses the logger when no warn callback is given", () => { + touch(legacyDb()); + const loggerWarn = vi.fn(); + vi.spyOn(loggerModule, "getLogger").mockReturnValue({ + warn: loggerWarn, + } as unknown as ReturnType<typeof loggerModule.getLogger>); + resolveDatabasePath(); + expect(loggerWarn).toHaveBeenCalledTimes(1); + }); + + it("does not warn when the workspace database already exists", () => { + touch(legacyDb()); + touch(wsDb("default")); + const warn = vi.fn(); + resolveDatabasePath({ warn }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("does not warn when an explicit path is set", () => { + touch(legacyDb()); + const warn = vi.fn(); + resolveDatabasePath({ explicitPath: legacyDb(), warn }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("does not warn when there is no legacy database", () => { + const warn = vi.fn(); + resolveDatabasePath({ warn }); + expect(warn).not.toHaveBeenCalled(); + }); + }); + + describe("config database.path", () => { + it("is unset by default so the workspace database is used", () => { + expect(loadConfig().database.path).toBeUndefined(); + }); + + it("expands ~ from the user config file", () => { + writeUserConfig({ database: { path: "~/custom/libscope.db" } }); + expect(loadConfig().database.path).toBe(join(tempHome, "custom", "libscope.db")); + }); + + it("project file wins over user file", () => { + writeUserConfig({ database: { path: "/user/libscope.db" } }); + writeFileSync( + join(tempCwd, ".libscope.json"), + JSON.stringify({ database: { path: "/project/libscope.db" } }), + ); + expect(loadConfig().database.path).toBe("/project/libscope.db"); + }); + }); + + describe("LibScope.create", () => { + const local = { embedding: { provider: "local" } } as const; + + it("opens the active workspace database (same as CLI/MCP)", async () => { + const ls = LibScope.create({ config: local }); + try { + expect(existsSync(wsDb("default"))).toBe(true); + expect(existsSync(legacyDb())).toBe(false); + const { stats } = await ls.overview(); + expect(stats.databaseSizeBytes).toBe(statSync(wsDb("default")).size); + expect(stats.databaseSizeBytes).toBeGreaterThan(0); + } finally { + ls.close(); + } + }); + + it("honours the workspace option", () => { + const ls = LibScope.create({ workspace: "sdk-ws", config: local }); + ls.close(); + expect(existsSync(wsDb("sdk-ws"))).toBe(true); + }); + + it("honours dbPath over workspace", () => { + const dbPath = join(tempHome, "explicit", "kb.db"); + const ls = LibScope.create({ dbPath, workspace: "ignored", config: local }); + ls.close(); + expect(existsSync(dbPath)).toBe(true); + expect(existsSync(wsDb("ignored"))).toBe(false); + }); + + it("honours database.path from a config file", () => { + writeUserConfig({ database: { path: "~/from-config.db" } }); + const ls = LibScope.create({ config: local }); + ls.close(); + expect(existsSync(join(tempHome, "from-config.db"))).toBe(true); + }); + }); + + describe("getStats", () => { + it("measures the open database file by default", () => { + const path = join(tempHome, "stats.db"); + const db = createDatabase(path); + try { + runMigrations(db); + const stats = getStats(db); + expect(stats.databaseSizeBytes).toBe(statSync(path).size); + expect(stats.databaseSizeBytes).toBeGreaterThan(0); + } finally { + db.close(); + } + }); + }); +}); diff --git a/tests/unit/docs-connector.test.ts b/tests/unit/docs-connector.test.ts index 9a974179..510da342 100644 --- a/tests/unit/docs-connector.test.ts +++ b/tests/unit/docs-connector.test.ts @@ -252,6 +252,12 @@ describe("extractDocTitle", () => { "https://example.com/docs", "My Library — Docs", ], + [ + "character references decoded", + "<html><head><title>Guide — Q&A", + "https://example.com/docs", + "Guide — Q&A", + ], [ "URL-derived fallback", "

content

", @@ -276,6 +282,18 @@ describe("extractDocTitle", () => { "https://example.com/", "example.com", ], + [ + "H1 with character references", + "

Tom & Jerry — API

", + "https://example.com/docs/api", + "Tom & Jerry — API", + ], + [ + " with character references", + "<html><head><title>textwrap — Python", + "https://example.com/docs", + "textwrap — Python", + ], [ "H1 precedence over title", "Page Title

Real Title

", @@ -341,7 +359,7 @@ describe("extractDocLinks", () => { 'Three', ].join("\n"); const links = extractDocLinks(html, BASE, "/docs/"); - expect(links.filter((l) => l.includes("/docs/page")).length).toBe(1); + expect(links.filter((l) => l.includes("/docs/page"))).toHaveLength(1); }); it("returns empty array when no anchors found", () => { @@ -402,7 +420,7 @@ describe("extractSitemapUrls", () => { https://docs.example.com/docs/page `; const urls = extractSitemapUrls(xml, BASE, "/docs/"); - expect(urls.length).toBe(1); + expect(urls).toHaveLength(1); }); it("returns empty array for empty sitemap", () => { @@ -796,6 +814,59 @@ describe("syncDocSite — mocked fetch", () => { expect(result2.pagesUpdated).toBe(0); }); + it("recognises existing pages beyond the first 50 documents", async () => { + mockFetch + .mockResolvedValueOnce(htmlResponse(SPHINX_ROOT_SIMPLE)) // root + .mockResolvedValueOnce(notFoundResponse()); // sitemap.xml + await syncDocSite(db, provider, { url: "https://docs.example.com/docs/" }); + + // Make the root doc the oldest of 61 docs-connector documents. + db.prepare("UPDATE documents SET updated_at = '2000-01-01 00:00:00'").run(); + const insert = db.prepare( + "INSERT INTO documents (id, source_type, title, content, url, updated_at) VALUES (?, 'library', 'T', 'B', ?, '2030-01-01 00:00:00')", + ); + for (let i = 0; i < 60; i++) { + insert.run(`filler-${i}`, `https://docs.example.com/docs/filler-${i}`); + } + + mockFetch + .mockResolvedValueOnce(htmlResponse(SPHINX_ROOT_SIMPLE)) // root (unchanged) + .mockResolvedValueOnce(notFoundResponse()); // sitemap.xml + const result = await syncDocSite(db, provider, { url: "https://docs.example.com/docs/" }); + + expect(result.pagesSkipped).toBe(1); + expect(result.pagesIndexed).toBe(0); + }); + + it("stops crawling when the abort signal fires", async () => { + const controller = new AbortController(); + const root = SPHINX_ROOT_SIMPLE.replace( + "", + 'P1', + ); + mockFetch + .mockImplementationOnce(() => { + controller.abort(); + return Promise.resolve(htmlResponse(root)); + }) + .mockResolvedValueOnce(notFoundResponse()); // sitemap.xml + + await expect( + syncDocSite( + db, + provider, + { url: "https://docs.example.com/docs/" }, + { + signal: controller.signal, + }, + ), + ).rejects.toThrow(); + + expect(mockFetch).toHaveBeenCalledTimes(2); // root + sitemap, never p1 + const row = db.prepare("SELECT status FROM connector_syncs").get() as { status: string }; + expect(row.status).toBe("failed"); + }); + it("records sync history in the connector_syncs table", async () => { mockFetch .mockResolvedValueOnce(htmlResponse(SPHINX_ROOT_SIMPLE)) // root @@ -853,13 +924,13 @@ describe("disconnectDocSite", () => { it("removes all documents from the given site URL prefix", () => { // Seed some docs manually db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', ?, ?, ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', ?, ?, ?, 'crawler')", ).run("doc-1", "Page 1", "Content 1", "https://docs.example.com/docs/page1"); db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', ?, ?, ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', ?, ?, ?, 'crawler')", ).run("doc-2", "Page 2", "Content 2", "https://docs.example.com/docs/page2"); db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', ?, ?, ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', ?, ?, ?, 'crawler')", ).run("doc-3", "Other", "Content 3", "https://other.example.com/docs/page"); const removed = disconnectDocSite(db, "https://docs.example.com/docs/"); @@ -870,6 +941,39 @@ describe("disconnectDocSite", () => { expect(remaining.n).toBe(1); // doc-3 should remain }); + it("keeps documents under the site URL that the connector did not index", () => { + const insert = db.prepare( + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, ?, ?, ?, ?, ?)", + ); + insert.run("synced", "library", "Synced", "Body", "https://docs.example.com/docs/a", "crawler"); + insert.run("added", "manual", "Added", "Body", "https://docs.example.com/docs/a/", "manual"); + insert.run("by-model", "library", "Model", "Body", "https://docs.example.com/docs/b", "model"); + + const removed = disconnectDocSite(db, "https://docs.example.com/docs/"); + + expect(removed).toBe(1); + const ids = ( + db.prepare("SELECT id FROM documents ORDER BY id").all() as Array<{ id: string }> + ).map((r) => r.id); + expect(ids).toEqual(["added", "by-model"]); + }); + + it("matches the site URL exactly, not as a LIKE pattern", () => { + const insert = db.prepare( + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', 'T', 'B', ?, 'crawler')", + ); + insert.run("inside", "https://docs.example.com/my_docs/page"); + insert.run("lookalike", "https://docs.example.com/myXdocs/page"); + + const removed = disconnectDocSite(db, "https://docs.example.com/my_docs/"); + + expect(removed).toBe(1); + const ids = (db.prepare("SELECT id FROM documents").all() as Array<{ id: string }>).map( + (r) => r.id, + ); + expect(ids).toEqual(["lookalike"]); + }); + it("returns 0 when no matching documents exist", () => { const removed = disconnectDocSite(db, "https://docs.example.com/docs/"); expect(removed).toBe(0); @@ -881,7 +985,7 @@ describe("disconnectDocSite", () => { it("does not remove documents from other sites", () => { db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', ?, ?, ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', ?, ?, ?, 'crawler')", ).run("doc-1", "Page 1", "Content 1", "https://other.example.com/docs/page"); const removed = disconnectDocSite(db, "https://docs.example.com/docs/"); @@ -893,7 +997,7 @@ describe("disconnectDocSite", () => { it("removes associated chunks", () => { db.prepare( - "INSERT INTO documents (id, source_type, title, content, url) VALUES (?, 'library', 'Title', 'Body', ?)", + "INSERT INTO documents (id, source_type, title, content, url, submitted_by) VALUES (?, 'library', 'Title', 'Body', ?, 'crawler')", ).run("doc-1", "https://docs.example.com/docs/page"); db.prepare( "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", diff --git a/tests/unit/embedding-dimensions.test.ts b/tests/unit/embedding-dimensions.test.ts new file mode 100644 index 00000000..4a871751 --- /dev/null +++ b/tests/unit/embedding-dimensions.test.ts @@ -0,0 +1,191 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +const openaiCreate = vi.fn(); + +vi.mock("openai", () => ({ + default: class { + embeddings = { create: openaiCreate }; + }, +})); + +vi.mock("../../src/utils/retry.js", () => ({ + withRetry: (fn: () => Promise): Promise => fn(), +})); + +import { + EmbeddingDimensions, + knownModelDimensions, + validateDimensionsOverride, +} from "../../src/providers/dimensions.js"; +import { OllamaEmbeddingProvider } from "../../src/providers/ollama.js"; +import { OpenAIEmbeddingProvider } from "../../src/providers/openai.js"; +import { createEmbeddingProvider } from "../../src/providers/index.js"; +import type { LibScopeConfig } from "../../src/config.js"; +import { ConfigError, EmbeddingError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; + +function vector(size: number): number[] { + return Array.from({ length: size }, () => 0.1); +} + +function config(embedding: LibScopeConfig["embedding"], openaiApiKey?: string): LibScopeConfig { + return { + embedding, + ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}), + database: { path: ":memory:" }, + indexing: { maxDocumentSize: 1024, allowPrivateUrls: false, allowSelfSignedCerts: false }, + logging: { level: "silent" }, + }; +} + +/** Make fetch return an Ollama /api/embed response with vectors of the given sizes. */ +function stubOllama(...sizes: number[]): void { + vi.stubGlobal( + "fetch", + vi.fn(() => + Promise.resolve( + new Response(JSON.stringify({ embeddings: sizes.map((s) => vector(s)) }), { status: 200 }), + ), + ), + ); +} + +beforeEach(() => { + initLogger("silent"); + openaiCreate.mockReset(); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("knownModelDimensions", () => { + it("knows common OpenAI and Ollama models", () => { + expect(knownModelDimensions("text-embedding-3-small")).toBe(1536); + expect(knownModelDimensions("text-embedding-3-large")).toBe(3072); + expect(knownModelDimensions("nomic-embed-text")).toBe(768); + expect(knownModelDimensions("mxbai-embed-large")).toBe(1024); + expect(knownModelDimensions("all-minilm")).toBe(384); + }); + + it("ignores Ollama tags", () => { + expect(knownModelDimensions("nomic-embed-text:v1.5")).toBe(768); + expect(knownModelDimensions("mxbai-embed-large:latest")).toBe(1024); + }); + + it("returns undefined for unknown models", () => { + expect(knownModelDimensions("my-custom-embedder")).toBeUndefined(); + }); +}); + +describe("validateDimensionsOverride", () => { + it("accepts undefined and positive integers", () => { + expect(validateDimensionsOverride(undefined)).toBeUndefined(); + expect(validateDimensionsOverride(1024)).toBe(1024); + }); + + it.each([0, -1, 1.5, 10001, "512"])("rejects %j", (value) => { + expect(() => validateDimensionsOverride(value)).toThrow(ConfigError); + }); +}); + +describe("EmbeddingDimensions", () => { + it("learns the size from the first vector and checks later vectors against it", () => { + const dims = new EmbeddingDimensions("custom"); + expect(dims.value).toBe(0); + dims.check([vector(5)]); + expect(dims.value).toBe(5); + expect(() => dims.check([vector(4)])).toThrow("Expected embedding dimension 5, got 4"); + }); + + it("prefers the override over the known size", () => { + expect(new EmbeddingDimensions("text-embedding-3-large", 256).value).toBe(256); + }); +}); + +describe("OllamaEmbeddingProvider dimensions", () => { + it("uses the known size of the model", () => { + expect(new OllamaEmbeddingProvider("http://x", "mxbai-embed-large").dimensions).toBe(1024); + }); + + it("learns the size of an unknown model from the first response", async () => { + const provider = new OllamaEmbeddingProvider("http://x", "custom-embed"); + expect(provider.dimensions).toBe(0); + stubOllama(640); + await provider.embed("hello"); + expect(provider.dimensions).toBe(640); + }); + + it("checks batch responses against the learned size", async () => { + const provider = new OllamaEmbeddingProvider("http://x", "custom-embed"); + stubOllama(32, 16); + await expect(provider.embedBatch(["a", "b"])).rejects.toThrow(EmbeddingError); + }); + + it("explains how to fix a wrong override", async () => { + const provider = new OllamaEmbeddingProvider("http://x", "custom-embed", 512); + stubOllama(1024); + await expect(provider.embed("hello")).rejects.toThrow( + /Expected embedding dimension 512, got 1024.*set embedding\.dimensions to 1024/, + ); + }); +}); + +describe("OpenAIEmbeddingProvider dimensions", () => { + it("uses the known size of the model", () => { + expect(new OpenAIEmbeddingProvider("k", "text-embedding-3-large").dimensions).toBe(3072); + }); + + it("sends the override as the dimensions parameter for text-embedding-3 models", async () => { + openaiCreate.mockResolvedValue({ data: [{ embedding: vector(256) }] }); + const provider = new OpenAIEmbeddingProvider("k", "text-embedding-3-large", 256); + + await provider.embed("hello"); + + expect(openaiCreate).toHaveBeenCalledWith( + expect.objectContaining({ model: "text-embedding-3-large", dimensions: 256 }), + ); + expect(provider.dimensions).toBe(256); + }); + + it("does not send a dimensions parameter without an override", async () => { + openaiCreate.mockResolvedValue({ + data: [{ embedding: vector(1536) }, { embedding: vector(1536) }], + }); + const provider = new OpenAIEmbeddingProvider("k", "text-embedding-3-small"); + + await provider.embedBatch(["a", "b"]); + + expect(openaiCreate.mock.calls[0]?.[0]).not.toHaveProperty("dimensions"); + }); +}); + +describe("createEmbeddingProvider dimensions", () => { + it("passes embedding.dimensions to the Ollama provider", () => { + const provider = createEmbeddingProvider( + config({ + provider: "ollama", + url: "http://x", + model: "custom-embed", + dimensions: 2048, + }), + ); + expect(provider.dimensions).toBe(2048); + expect(provider.model).toBe("custom-embed"); + }); + + it("uses the known size for the configured OpenAI model", () => { + const provider = createEmbeddingProvider( + config({ provider: "openai", model: "text-embedding-3-large" }, "k"), + ); + expect(provider.dimensions).toBe(3072); + }); + + it("rejects an invalid embedding.dimensions", () => { + expect(() => + createEmbeddingProvider( + config({ provider: "ollama", url: "http://x", model: "m", dimensions: -5 }), + ), + ).toThrow(ConfigError); + }); +}); diff --git a/tests/unit/embedding-text-consistency.test.ts b/tests/unit/embedding-text-consistency.test.ts new file mode 100644 index 00000000..9417319c --- /dev/null +++ b/tests/unit/embedding-text-consistency.test.ts @@ -0,0 +1,177 @@ +import { describe, it, expect, beforeAll, afterEach } from "vitest"; +import { writeFileSync, mkdtempSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { + buildEmbeddingText, + indexDocument, + type IndexDocumentInput, +} from "../../src/core/indexing.js"; +import { updateDocument } from "../../src/core/documents.js"; +import { reindex } from "../../src/core/reindex.js"; +import { installPack } from "../../src/core/packs.js"; +import { getDocumentLinks } from "../../src/core/links.js"; + +/** Mock provider that records every text it is asked to embed. */ +class CapturingProvider extends MockEmbeddingProvider { + inputs: string[] = []; + + override embedBatch(texts: string[]): Promise { + this.inputs.push(...texts); + return super.embedBatch(texts); + } +} + +const CONTENT = "# Setup\n\nInstall the package.\n\n## Usage\n\nCall the function."; +const DOC: IndexDocumentInput = { + title: "Widget Guide", + content: CONTENT, + sourceType: "library", + library: "widgets", + version: "2.1.0", +}; + +/** Embedding inputs and stored vectors produced by a fresh indexDocument call. */ +async function freshIndex( + input: IndexDocumentInput, +): Promise<{ inputs: string[]; vectors: Buffer[] }> { + const db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const { id } = await indexDocument(db, provider, input); + const vectors = storedVectors(db, id); + db.close(); + return { inputs: provider.inputs, vectors }; +} + +function storedVectors(db: Database.Database, docId: string): Buffer[] { + return ( + db + .prepare( + `SELECT e.embedding FROM chunks c JOIN chunk_embeddings e ON e.chunk_id = c.id + WHERE c.document_id = ? ORDER BY c.chunk_index`, + ) + .all(docId) as Array<{ embedding: Buffer }> + ).map((r) => r.embedding); +} + +describe("embedding text is the same on every write path", () => { + let db: Database.Database | undefined; + + beforeAll(() => { + initLogger("silent"); + }); + + afterEach(() => { + db?.close(); + db = undefined; + }); + + it("buildEmbeddingText prefixes title, library, and version", () => { + expect(buildEmbeddingText("body", { title: "T", library: "L", version: "1" })).toBe( + "T | Library: L | Version: 1\n\nbody", + ); + expect(buildEmbeddingText("body", { title: "T", library: null, version: undefined })).toBe( + "T\n\nbody", + ); + expect(buildEmbeddingText("body", {})).toBe("body"); + }); + + it("reindex embeds the same text and stores the same vectors as a fresh index", async () => { + const fresh = await freshIndex(DOC); + expect(fresh.inputs[0]).toMatch(/^Widget Guide \| Library: widgets \| Version: 2\.1\.0\n\n/); + + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const { id } = await indexDocument(db, provider, DOC); + provider.inputs = []; + await reindex(db, provider, { documentIds: [id] }); + + expect(provider.inputs).toEqual(fresh.inputs); + expect(storedVectors(db, id)).toEqual(fresh.vectors); + }); + + it("updateDocument with new content embeds the same text as a fresh index", async () => { + const fresh = await freshIndex(DOC); + + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const { id } = await indexDocument(db, provider, { ...DOC, content: "Old content." }); + provider.inputs = []; + await updateDocument(db, provider, id, { content: CONTENT }); + + expect(provider.inputs).toEqual(fresh.inputs); + expect(storedVectors(db, id)).toEqual(fresh.vectors); + }); + + it("updateDocument with new title/library/version re-embeds to match a fresh index", async () => { + const renamed = { ...DOC, title: "Widget Handbook", library: "gadgets", version: "3.0.0" }; + const fresh = await freshIndex(renamed); + + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const { id } = await indexDocument(db, provider, DOC); + provider.inputs = []; + await updateDocument(db, provider, id, { + title: renamed.title, + metadata: { library: renamed.library, version: renamed.version }, + }); + + expect(provider.inputs).toEqual(fresh.inputs); + expect(storedVectors(db, id)).toEqual(fresh.vectors); + }); + + it("installPack embeds the same text as a fresh index of the same document", async () => { + const fresh = await freshIndex({ title: DOC.title, content: CONTENT, sourceType: "library" }); + + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const dir = mkdtempSync(join(tmpdir(), "libscope-embed-text-")); + const packPath = join(dir, "pack.json"); + writeFileSync( + packPath, + JSON.stringify({ + name: "widgets-pack", + version: "1.0.0", + description: "test", + documents: [{ title: DOC.title, content: CONTENT, source: "" }], + metadata: { author: "a", license: "MIT", createdAt: "2024-01-01" }, + }), + ); + await installPack(db, provider, packPath); + const { id } = db + .prepare("SELECT id FROM documents WHERE pack_name = ?") + .get("widgets-pack") as { + id: string; + }; + + expect(provider.inputs).toEqual(fresh.inputs); + expect(storedVectors(db, id)).toEqual(fresh.vectors); + }); + + it("updateDocument with new content extracts document links like indexDocument", async () => { + db = createTestDbWithVec(); + const provider = new CapturingProvider(); + const target = await indexDocument(db, provider, { + title: "Target Doc", + content: "Target body.", + sourceType: "manual", + }); + const { id } = await indexDocument(db, provider, { + title: "Source Doc", + content: "No links yet.", + sourceType: "manual", + }); + expect(getDocumentLinks(db, id).outgoing).toHaveLength(0); + + await updateDocument(db, provider, id, { content: "See [[Target Doc]] for details." }); + + const outgoing = getDocumentLinks(db, id).outgoing; + expect(outgoing).toHaveLength(1); + expect(outgoing[0]!.targetId).toBe(target.id); + expect(outgoing[0]!.linkType).toBe("references"); + }); +}); diff --git a/tests/unit/errors.test.ts b/tests/unit/errors.test.ts index 3170ec72..dced1a45 100644 --- a/tests/unit/errors.test.ts +++ b/tests/unit/errors.test.ts @@ -8,8 +8,27 @@ import { ConfigError, DocumentNotFoundError, ChunkNotFoundError, + TopicNotFoundError, + NotFoundError, } from "../../src/errors.js"; +describe("NotFoundError", () => { + it("is the base class of every not-found error and keeps each code", () => { + const cases: Array<[NotFoundError, string]> = [ + [new DocumentNotFoundError("d"), "DOCUMENT_NOT_FOUND"], + [new ChunkNotFoundError("c"), "CHUNK_NOT_FOUND"], + [new TopicNotFoundError("t"), "TOPIC_NOT_FOUND"], + [new NotFoundError("Link not found: l", "LINK_NOT_FOUND"), "LINK_NOT_FOUND"], + [new NotFoundError("gone"), "NOT_FOUND"], + ]; + for (const [err, code] of cases) { + expect(err).toBeInstanceOf(NotFoundError); + expect(err).toBeInstanceOf(LibScopeError); + expect(err.code).toBe(code); + } + }); +}); + describe("errors", () => { it("should create LibScopeError with code", () => { const err = new LibScopeError("test", "TEST_ERROR"); diff --git a/tests/unit/glob-edge-cases.test.ts b/tests/unit/glob-edge-cases.test.ts new file mode 100644 index 00000000..c35af760 --- /dev/null +++ b/tests/unit/glob-edge-cases.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; +import { globToRegExp, toPosixPath } from "../../src/utils/glob.js"; + +describe("globToRegExp edge cases", () => { + it.each([ + ["**/ also matches zero directories", "**/a.md", "^(?:.*/)?a\\.md$"], + ["** without a slash", "a/**", "^a/.*$"], + ["a single *", "*.md", "^[^/]*\\.md$"], + ["? as one character", "a?", "^a[^/]$"], + ["{ inside braces is literal", "{a{b}", "^(?:a\\{b)$"], + ["} outside braces is literal", "a}", "^a\\}$"], + [", outside braces is literal", "a,b", "^a,b$"], + ["an unclosed brace group is closed", "{a,b", "^(?:a|b)$"], + ["a backslash is escaped", "a\\b", "^a\\\\b$"], + ])("%s", (_case, glob, source) => { + expect(globToRegExp(glob).source).toBe(new RegExp(source).source); + }); +}); + +describe("toPosixPath edge cases", () => { + it("leaves forward slashes alone and converts every backslash", () => { + expect(toPosixPath("a/b\\c\\\\d")).toBe("a/b/c//d"); + }); +}); diff --git a/tests/unit/glob.test.ts b/tests/unit/glob.test.ts new file mode 100644 index 00000000..36a5a826 --- /dev/null +++ b/tests/unit/glob.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect } from "vitest"; +import { globToRegExp, toPosixPath } from "../../src/utils/glob.js"; + +describe("globToRegExp", () => { + it("matches files at any depth with **/ including the root", () => { + const rx = globToRegExp("**/*.md"); + expect(rx.test("readme.md")).toBe(true); + expect(rx.test("docs/guide/intro.md")).toBe(true); + expect(rx.test("docs/intro.txt")).toBe(false); + }); + + it("keeps * within a single path segment", () => { + const rx = globToRegExp("docs/*.md"); + expect(rx.test("docs/a.md")).toBe(true); + expect(rx.test("docs/sub/a.md")).toBe(false); + }); + + it("supports brace alternation", () => { + const rx = globToRegExp("**/*.{md,mdx,txt}"); + expect(rx.test("a.md")).toBe(true); + expect(rx.test("x/y/b.mdx")).toBe(true); + expect(rx.test("c.txt")).toBe(true); + expect(rx.test("d.pdf")).toBe(false); + }); + + it("supports ? and trailing **", () => { + expect(globToRegExp("file?.txt").test("file1.txt")).toBe(true); + expect(globToRegExp("file?.txt").test("file12.txt")).toBe(false); + expect(globToRegExp("node_modules/**").test("node_modules/a/b.js")).toBe(true); + }); + + it("escapes regex metacharacters", () => { + const rx = globToRegExp("a+b(1).md"); + expect(rx.test("a+b(1).md")).toBe(true); + expect(rx.test("aab1.md")).toBe(false); + }); + + it("closes an unterminated brace group", () => { + expect(globToRegExp("*.{md,txt").test("a.txt")).toBe(true); + }); +}); + +describe("toPosixPath", () => { + it("converts backslashes to forward slashes", () => { + expect(toPosixPath(String.raw`docs\guide\a.md`)).toBe("docs/guide/a.md"); + }); +}); diff --git a/tests/unit/html-entities.test.ts b/tests/unit/html-entities.test.ts new file mode 100644 index 00000000..106065f5 --- /dev/null +++ b/tests/unit/html-entities.test.ts @@ -0,0 +1,21 @@ +import { describe, it, expect } from "vitest"; +import { decodeHtmlEntities } from "../../src/core/html-entities.js"; + +describe("decodeHtmlEntities", () => { + it.each([ + ["decimal reference", "textwrap — Python docs", "textwrap — Python docs"], + ["hex reference", "A — B A", "A — B A"], + [ + "named references", + "Tom & Jerry <3 "hi" — ok", + 'Tom & Jerry <3 "hi" — ok', + ], + ["named reference in upper case", "A & B", "A & B"], + ["decodes only once", "&lt;", "<"], + ["unknown named reference kept", "&unknownent; x", "&unknownent; x"], + ["out-of-range code point kept", "� �", "� �"], + ["text without references", "Plain title", "Plain title"], + ])("%s", (_label, input, expected) => { + expect(decodeHtmlEntities(input)).toBe(expected); + }); +}); diff --git a/tests/unit/index-meta.test.ts b/tests/unit/index-meta.test.ts new file mode 100644 index 00000000..aa360850 --- /dev/null +++ b/tests/unit/index-meta.test.ts @@ -0,0 +1,231 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import type Database from "better-sqlite3"; +import DatabaseConstructor from "better-sqlite3"; +import { createDatabase } from "../../src/db/connection.js"; +import { runMigrations, createVectorTable } from "../../src/db/schema.js"; +import { + getVectorTableDimensions, + readEmbeddingIdentity, + rebuildVectorTable, +} from "../../src/db/index-meta.js"; +import { searchDocuments } from "../../src/core/search.js"; +import { ConfigError, DatabaseError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; +import type { EmbeddingProvider } from "../../src/providers/embedding.js"; + +/** Provider stub that returns constant vectors of a fixed size. */ +class FixedProvider implements EmbeddingProvider { + embedCalls = 0; + constructor( + readonly name: string, + readonly model: string, + private size: number, + private readonly reportedSize: number = size, + ) {} + + get dimensions(): number { + return this.reportedSize === 0 && this.embedCalls > 0 ? this.size : this.reportedSize; + } + + embed(_text: string): Promise { + this.embedCalls++; + return Promise.resolve(Array.from({ length: this.size }, (_, i) => (i + 1) / this.size)); + } + + async embedBatch(texts: string[]): Promise { + return Promise.all(texts.map((t) => this.embed(t))); + } +} + +function hasSqliteVec(): boolean { + const db = createDatabase(":memory:"); + try { + db.prepare("SELECT vec_version()").get(); + return true; + } catch { + return false; + } finally { + db.close(); + } +} + +const vecAvailable = hasSqliteVec(); + +function vecRowCount(db: Database.Database): number { + return (db.prepare("SELECT COUNT(*) AS n FROM chunk_embeddings").get() as { n: number }).n; +} + +describe.skipIf(!vecAvailable)("vector index metadata (sqlite-vec)", () => { + let db: Database.Database; + + beforeEach(() => { + initLogger("silent"); + db = createDatabase(":memory:"); + }); + + afterEach(() => { + db.close(); + }); + + it("records provider, model and dimensions when it creates the vector table", () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("ollama", "nomic-embed-text", 768)); + + expect(getVectorTableDimensions(db)).toBe(768); + expect(readEmbeddingIdentity(db)).toEqual({ + provider: "ollama", + model: "nomic-embed-text", + dimensions: 768, + }); + }); + + it("populates metadata for a vector table built before schema v18", () => { + runMigrations(db, 17); + db.exec( + "CREATE VIRTUAL TABLE chunk_embeddings USING vec0(chunk_id TEXT PRIMARY KEY, embedding float[4])", + ); + runMigrations(db); + expect(readEmbeddingIdentity(db)).toEqual({ + provider: undefined, + model: undefined, + dimensions: undefined, + }); + + createVectorTable(db, 4); + expect(readEmbeddingIdentity(db).dimensions).toBe(4); + + createVectorTable(db, new FixedProvider("mock", "m1", 4)); + expect(readEmbeddingIdentity(db)).toEqual({ provider: "mock", model: "m1", dimensions: 4 }); + }); + + it("rejects a provider whose vector size differs from the index", () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("ollama", "nomic-embed-text", 768)); + + const change = (): void => + createVectorTable(db, new FixedProvider("ollama", "mxbai-embed-large", 1024)); + expect(change).toThrow(ConfigError); + expect(change).toThrow(/vector index \(ollama\/nomic-embed-text, 768 dimensions\)/); + expect(change).toThrow( + /configured embedding model \(ollama\/mxbai-embed-large, 1024 dimensions\)/, + ); + expect(change).toThrow(/libscope admin reindex --rebuild/); + // A bare vector size is checked against the table too. + expect(() => createVectorTable(db, 1024)).toThrow( + /does not match the configured embedding model \(1024 dimensions\)/, + ); + }); + + it("rejects a different model with the same vector size", () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("openai", "text-embedding-3-small", 1536)); + + expect(() => + createVectorTable(db, new FixedProvider("openai", "text-embedding-ada-002", 1536)), + ).toThrow(/text-embedding-3-small.*text-embedding-ada-002/); + }); + + it("accepts the same provider again and a provider whose size is not known yet", () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("ollama", "custom", 512)); + + expect(() => createVectorTable(db, new FixedProvider("ollama", "custom", 512))).not.toThrow(); + expect(() => + createVectorTable(db, new FixedProvider("ollama", "custom", 512, 0)), + ).not.toThrow(); + expect(() => createVectorTable(db, 512)).not.toThrow(); + }); + + it("refuses to create a table when the vector size is unknown", () => { + runMigrations(db); + const unknown = (): void => + createVectorTable(db, new FixedProvider("ollama", "custom", 512, 0)); + expect(unknown).toThrow(DatabaseError); + expect(unknown).toThrow(/embedding\.dimensions/); + }); + + it("rebuildVectorTable recreates the table with the new size and records it", async () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("ollama", "nomic-embed-text", 4)); + db.prepare("INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)").run( + "c1", + Buffer.from(new Float32Array([1, 0, 0, 0]).buffer), + ); + + const next = new FixedProvider("ollama", "mxbai-embed-large", 8); + const identity = await rebuildVectorTable(db, next); + + expect(identity).toEqual({ provider: "ollama", model: "mxbai-embed-large", dimensions: 8 }); + expect(getVectorTableDimensions(db)).toBe(8); + expect(vecRowCount(db)).toBe(0); + expect(readEmbeddingIdentity(db)).toEqual(identity); + expect(() => createVectorTable(db, next)).not.toThrow(); + }); + + it("rebuildVectorTable embeds a probe text when the provider size is unknown", async () => { + runMigrations(db); + const provider = new FixedProvider("ollama", "custom", 6, 0); + + const identity = await rebuildVectorTable(db, provider); + + expect(provider.embedCalls).toBe(1); + expect(identity.dimensions).toBe(6); + expect(getVectorTableDimensions(db)).toBe(6); + }); + + it("rebuildVectorTable keeps the old table when the model output does not match its size", async () => { + runMigrations(db); + createVectorTable(db, 4); + // Reports 12 dimensions but returns 40-dimension vectors (a wrong embedding.dimensions). + const provider = new FixedProvider("ollama", "custom", 40, 12); + + await expect(rebuildVectorTable(db, provider)).rejects.toThrow( + "Expected embedding dimension 12, got 40", + ); + expect(getVectorTableDimensions(db)).toBe(4); + }); + + it("search falls back to keyword results when the query vector size does not match", async () => { + runMigrations(db); + createVectorTable(db, 4); + db.prepare( + "INSERT INTO documents (id, source_type, title, content) VALUES ('d1', 'manual', 'Doc', 'x')", + ).run(); + db.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES ('c1', 'd1', 'kiwi orchard', 0)", + ).run(); + db.prepare("INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, ?)").run( + "c1", + Buffer.from(new Float32Array([1, 0, 0, 0]).buffer), + ); + + const { results } = await searchDocuments(db, new FixedProvider("mock", "m", 3), { + query: "kiwi", + }); + + expect(results.map((r) => r.chunkId)).toEqual(["c1"]); + expect(results[0]?.scoreExplanation.method).not.toBe("vector"); + }); +}); + +describe("vector index metadata (no sqlite-vec)", () => { + beforeEach(() => { + initLogger("silent"); + }); + + it("rebuildVectorTable reports that sqlite-vec is required", async () => { + const db = new DatabaseConstructor(":memory:"); + runMigrations(db); + await expect(rebuildVectorTable(db, new FixedProvider("mock", "m", 4))).rejects.toThrow( + /sqlite-vec/, + ); + db.close(); + }); + + it("readEmbeddingIdentity returns nothing before migration 18", () => { + const db = new DatabaseConstructor(":memory:"); + runMigrations(db, 17); + expect(readEmbeddingIdentity(db)).toEqual({}); + db.close(); + }); +}); diff --git a/tests/unit/ingest.test.ts b/tests/unit/ingest.test.ts new file mode 100644 index 00000000..a5ee2b0f --- /dev/null +++ b/tests/unit/ingest.test.ts @@ -0,0 +1,241 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type Database from "better-sqlite3"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { ValidationError } from "../../src/errors.js"; + +vi.mock("../../src/core/url-fetcher.js", async (importOriginal) => { + const orig = await importOriginal>(); + return { + ...orig, + fetchAndConvert: vi.fn((url: string) => + Promise.resolve({ title: `Page ${url}`, content: `Fetched content of ${url}` }), + ), + }; +}); + +vi.mock("../../src/core/spider.js", () => ({ + // eslint-disable-next-line @typescript-eslint/require-await + spiderUrl: async function* ( + url: string, + ): AsyncGenerator< + { url: string; title: string; content: string; depth: number }, + { pagesFetched: number; pagesCrawled: number; pagesSkipped: number; errors: never[] } + > { + yield { url, title: "Seed", content: "Seed page content", depth: 0 }; + yield { url: `${url}/empty`, title: "Empty", content: " ", depth: 1 }; + return { pagesFetched: 2, pagesCrawled: 2, pagesSkipped: 1, errors: [] }; + }, +})); + +vi.mock("../../src/core/repo.js", async (importOriginal) => { + const orig = await importOriginal>(); + return { + ...orig, + indexRepository: vi.fn(() => + Promise.resolve({ + indexed: 1, + skipped: 2, + errors: ["bad.md: HTTP 404"], + documents: [{ documentId: "repo-doc", title: "README", chunkCount: 3, path: "README.md" }], + }), + ), + }; +}); + +const { detectIngestKind, ingest, listDirectoryFiles } = await import("../../src/core/ingest.js"); +const { resolveSourceType } = await import("../../src/core/indexing.js"); +const { isRepoUrl, indexRepository } = await import("../../src/core/repo.js"); +const { getDocumentTags } = await import("../../src/core/tags.js"); + +initLogger("silent"); + +describe("resolveSourceType", () => { + it("prefers explicit, then library, then topic, then manual", () => { + expect(resolveSourceType({ sourceType: "model-generated", library: "x" })).toBe( + "model-generated", + ); + expect(resolveSourceType({ library: "x", topic: "t" })).toBe("library"); + expect(resolveSourceType({ topic: "t" })).toBe("topic"); + expect(resolveSourceType({})).toBe("manual"); + }); +}); + +describe("isRepoUrl", () => { + it("recognises repository and tree URLs only", () => { + expect(isRepoUrl("https://github.com/owner/repo")).toBe(true); + expect(isRepoUrl("https://github.com/owner/repo.git")).toBe(true); + expect(isRepoUrl("https://github.com/owner/repo/tree/main/docs")).toBe(true); + expect(isRepoUrl("https://gitlab.com/owner/repo/-/tree/main")).toBe(true); + expect(isRepoUrl("https://github.com/owner/repo/blob/main/README.md")).toBe(false); + expect(isRepoUrl("https://github.com/owner")).toBe(false); + expect(isRepoUrl("https://example.com/owner/repo")).toBe(false); + expect(isRepoUrl("not a url")).toBe(false); + }); +}); + +describe("ingest", () => { + let db: Database.Database; + let provider: MockEmbeddingProvider; + let dir: string; + + beforeEach(() => { + db = createTestDb(); + provider = new MockEmbeddingProvider(); + dir = mkdtempSync(join(tmpdir(), "libscope-ingest-")); + }); + + afterEach(() => { + db.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + it("detects the kind of input", () => { + writeFileSync(join(dir, "a.md"), "# A"); + expect(detectIngestKind({ content: "x" })).toBe("content"); + expect(detectIngestKind({ source: "https://github.com/o/r" })).toBe("repo"); + expect(detectIngestKind({ source: "https://github.com/o/r", spider: true })).toBe("url"); + expect(detectIngestKind({ source: "https://example.com/docs" })).toBe("url"); + expect(detectIngestKind({ source: dir })).toBe("directory"); + expect(detectIngestKind({ source: join(dir, "a.md") })).toBe("file"); + expect(() => detectIngestKind({ source: join(dir, "missing.md") })).toThrow(ValidationError); + expect(() => detectIngestKind({})).toThrow(ValidationError); + }); + + it("lists directory files with include/exclude globs and skips hidden entries", () => { + mkdirSync(join(dir, "docs", "api"), { recursive: true }); + mkdirSync(join(dir, ".git")); + writeFileSync(join(dir, "README.md"), "x"); + writeFileSync(join(dir, "docs", "guide.md"), "x"); + writeFileSync(join(dir, "docs", "api", "ref.md"), "x"); + writeFileSync(join(dir, "docs", "app.min.js"), "x"); + writeFileSync(join(dir, ".git", "HEAD.md"), "x"); + writeFileSync(join(dir, ".hidden.md"), "x"); + + const rel = (files: string[]): string[] => files.map((f) => f.slice(dir.length + 1)); + expect(rel(listDirectoryFiles(dir))).toEqual([ + join("docs", "api", "ref.md"), + join("docs", "app.min.js"), + join("docs", "guide.md"), + "README.md", + ]); + expect( + rel(listDirectoryFiles(dir, { include: ["docs/**/*.md"], exclude: ["api/**"] })), + ).toEqual([join("docs", "api", "ref.md"), join("docs", "guide.md")]); + expect(rel(listDirectoryFiles(dir, { exclude: ["*.min.js", "docs/api/**"] }))).toEqual([ + join("docs", "guide.md"), + "README.md", + ]); + }); + + it("indexes a directory, skipping unsupported files, and applies tags", async () => { + writeFileSync(join(dir, "a.md"), "# A\n\nAlpha"); + writeFileSync(join(dir, "b.txt"), "Bravo text"); + writeFileSync(join(dir, "c.bin"), "binary"); + const progress: number[] = []; + const result = await ingest( + { db, provider, onProgress: (p) => progress.push(p.done) }, + { source: dir, library: "lib", tags: ["imported"] }, + ); + expect(result.kind).toBe("directory"); + expect(result.documents.map((d) => d.title)).toEqual(["a", "b"]); + expect(result.skipped).toEqual([ + { source: join(dir, "c.bin"), reason: "unsupported file format" }, + ]); + expect(progress).toEqual([1, 2]); + const id = result.documents[0]!.documentId; + expect(getDocumentTags(db, id).map((t) => t.name)).toEqual(["imported"]); + const row = db.prepare("SELECT source_type, library FROM documents WHERE id = ?").get(id); + expect(row).toEqual({ source_type: "library", library: "lib" }); + }); + + it("stores a file's absolute path as its URL, so adding it again replaces it", async () => { + const file = join(dir, "a.md"); + writeFileSync(file, "# A\n\nAlpha"); + const first = await ingest({ db, provider }, { source: file }); + const unchanged = await ingest({ db, provider }, { source: file }); + expect(unchanged.documents[0]?.documentId).toBe(first.documents[0]?.documentId); + writeFileSync(file, "# A\n\nAlpha, changed"); + await ingest({ db, provider }, { source: file }); + expect(db.prepare("SELECT url, content FROM documents").all()).toEqual([ + { url: file, content: "# A\n\nAlpha, changed" }, + ]); + }); + + it("plans without indexing on a dry run", async () => { + writeFileSync(join(dir, "a.md"), "# A"); + const result = await ingest({ db, provider }, { source: dir, dryRun: true }); + expect(result.planned).toEqual([join(dir, "a.md")]); + expect(db.prepare("SELECT COUNT(*) AS n FROM documents").get()).toEqual({ n: 0 }); + }); + + it("requires a title for inline content", async () => { + await expect(ingest({ db, provider }, { content: "body" })).rejects.toThrow(ValidationError); + const result = await ingest( + { db, provider }, + { content: "body", title: "T", url: "https://x.dev" }, + ); + expect(result.documents[0]).toMatchObject({ title: "T", source: "https://x.dev" }); + }); + + it("fetches a single URL and records who submitted it", async () => { + const result = await ingest( + { db, provider, submittedBy: "model" }, + { source: "https://example.com/page" }, + ); + expect(result.documents[0]?.title).toBe("Page https://example.com/page"); + const row = db.prepare("SELECT submitted_by, url FROM documents").get(); + expect(row).toEqual({ submitted_by: "model", url: "https://example.com/page" }); + }); + + it("crawls a URL, recording per-page errors and crawl stats", async () => { + const result = await ingest( + { db, provider }, + { source: "https://example.com/docs", spider: true }, + ); + expect(result.documents.map((d) => d.title)).toEqual(["Seed"]); + expect(result.errors.map((e) => e.source)).toEqual(["https://example.com/docs/empty"]); + expect(result.errors[0]?.error).toContain("content"); + expect(result.crawl).toEqual({ pagesCrawled: 2, pagesSkipped: 1 }); + }); + + it("indexes a repository as library docs and reports skipped files and errors", async () => { + const result = await ingest( + { db, provider }, + { source: "https://github.com/o/r", extensions: ["md"], topic: "t1" }, + ); + expect(vi.mocked(indexRepository)).toHaveBeenCalledWith( + db, + provider, + expect.objectContaining({ extensions: [".md"], topicId: "t1", sourceType: "library" }), + expect.any(Function), + ); + expect(result.documents).toEqual([ + { + documentId: "repo-doc", + title: "README", + chunkCount: 3, + source: "https://github.com/o/r#README.md", + }, + ]); + expect(result.errors).toEqual([ + { source: "https://github.com/o/r", error: "bad.md: HTTP 404" }, + ]); + expect(result.skipped).toEqual([ + { source: "https://github.com/o/r", reason: "2 unchanged file(s)" }, + ]); + }); + + it("stops between files when aborted", async () => { + writeFileSync(join(dir, "a.md"), "# A"); + const controller = new AbortController(); + controller.abort(new Error("stop")); + await expect( + ingest({ db, provider, signal: controller.signal }, { source: dir }), + ).rejects.toThrow("stop"); + }); +}); diff --git a/tests/unit/libscope.test.ts b/tests/unit/libscope.test.ts new file mode 100644 index 00000000..f1df0ef3 --- /dev/null +++ b/tests/unit/libscope.test.ts @@ -0,0 +1,342 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + expectTypeOf, + it, + vi, +} from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { LibScope, sdkOperationNames, type LibScopeOptions } from "../../src/LibScope.js"; +import { OPERATIONS } from "../../src/core/operations/index.js"; +import type { LlmProvider } from "../../src/core/rag.js"; +import { ConfigError, NotFoundError, ValidationError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; + +function create(options: LibScopeOptions = {}): LibScope { + return LibScope.create({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + useConfigFile: false, + ...options, + }); +} + +function mockLlm(text = "Mocked answer"): { llm: LlmProvider; complete: ReturnType } { + const complete = vi.fn().mockResolvedValue({ text, tokensUsed: 3 }); + return { llm: { model: "mock-llm", complete }, complete }; +} + +async function addDoc(scope: LibScope, title: string, content?: string): Promise { + const result = await scope.add({ + content: content ?? `${title} explains testing the LibScope SDK end to end`, + title, + }); + return result.documents[0]!.documentId; +} + +let tempHome: string; +let savedHome: string | undefined; + +beforeAll(() => { + initLogger("silent"); + savedHome = process.env["HOME"]; + tempHome = mkdtempSync(join(tmpdir(), "libscope-sdk-home-")); + process.env["HOME"] = tempHome; +}); + +afterAll(() => { + if (savedHome === undefined) delete process.env["HOME"]; + else process.env["HOME"] = savedHome; + rmSync(tempHome, { recursive: true, force: true }); +}); + +describe("LibScope SDK surface", () => { + it("reaches every operation exactly once", () => { + const names = sdkOperationNames(); + expect(new Set(names).size).toBe(names.length); + expect([...names].sort()).toEqual(OPERATIONS.map((op) => op.name).sort()); + }); + + it("types namespace methods from the operation schemas", () => { + const scope = create(); + try { + expectTypeOf(scope.docs.get).parameter(0).toMatchTypeOf<{ documentId: string }>(); + expectTypeOf(scope.tags.list).parameter(0).toEqualTypeOf | undefined>(); + expectTypeOf>["items"][number]>().toHaveProperty( + "chunkId", + ); + } finally { + scope.close(); + } + }); +}); + +describe("LibScope", () => { + let scope: LibScope; + + beforeEach(() => { + scope = create(); + }); + + afterEach(() => { + scope.close(); + }); + + it("add + search + overview", async () => { + const added = await scope.add({ content: "Vitest runs fast unit tests", title: "Vitest" }); + expect(added.kind).toBe("content"); + expect(added.documents[0]!.title).toBe("Vitest"); + + const byString = await scope.search("unit tests"); + const byObject = await scope.search({ query: "unit tests", limit: 5, diversity: 0.5 }); + expect(byString.items.length).toBeGreaterThan(0); + expect(byObject).toMatchObject({ limit: 5, offset: 0 }); + expect(byObject.total).toBeGreaterThan(0); + expect(byString.items[0]!.documentId).toBe(added.documents[0]!.documentId); + + const overview = await scope.overview(); + expect(overview.stats.totalDocuments).toBe(1); + expect(overview.health.database).toBe("ok"); + }); + + it("validates input with ValidationError", async () => { + await expect(scope.search({})).rejects.toBeInstanceOf(ValidationError); + await expect(scope.docs.get({ documentId: "" })).rejects.toBeInstanceOf(ValidationError); + }); + + it("docs: list, get, update, rate, history, rollback, delete", async () => { + const id = await addDoc(scope, "Doc One"); + const list = await scope.docs.list(); + expect(list.total).toBe(1); + expect(list.items[0]!.documentId).toBe(id); + + const view = await scope.docs.get({ documentId: id, maxLength: 5 }); + expect(view.content).toHaveLength(5); + expect(view.document.documentId).toBe(id); + + await scope.docs.update({ documentId: id, content: "Doc One rewritten content", tags: ["a"] }); + expect((await scope.docs.get({ documentId: id })).tags).toEqual(["a"]); + + const rating = await scope.docs.rate({ documentId: id, rating: 4 }); + expect(rating).toMatchObject({ documentId: id, rating: 4 }); + + const history = await scope.docs.history({ documentId: id }); + expect(history.items.length).toBeGreaterThan(0); + await scope.docs.rollback({ documentId: id, version: history.items[0]!.version }); + + await scope.docs.delete({ documentId: id }); + await expect(scope.docs.get({ documentId: id })).rejects.toBeInstanceOf(NotFoundError); + }); + + it("topics, tags and links", async () => { + const topic = await scope.topics.create({ name: "testing" }); + expect((await scope.topics.list()).items.map((t) => t.id)).toContain(topic.id); + + const a = await addDoc(scope, "Doc A"); + const b = await addDoc(scope, "Doc B"); + + expect((await scope.tags.add({ documentId: a, tags: ["x", "y"] })).tags).toEqual(["x", "y"]); + expect((await scope.tags.remove({ documentId: a, tags: ["y"] })).tags).toEqual(["x"]); + expect((await scope.tags.list()).items.map((t) => t.name)).toContain("x"); + expect((await scope.tags.suggest({ documentId: a })).documentId).toBe(a); + + const link = await scope.links.create({ + documentId: b, + targetDocumentId: a, + linkType: "prerequisite", + }); + expect((await scope.links.list({ documentId: b })).items).toHaveLength(1); + expect((await scope.links.prerequisites({ documentId: a })).items[0]!.documentId).toBe(b); + await scope.links.delete({ linkId: link.linkId }); + expect((await scope.links.list()).items).toHaveLength(0); + + await scope.topics.delete({ topic: "testing" }); + expect((await scope.topics.list()).items).toHaveLength(0); + }); + + it("searches: save, list, run, delete", async () => { + await addDoc(scope, "Saved Search Doc"); + await scope.searches.save({ name: "mine", query: "SDK" }); + expect((await scope.searches.list()).total).toBe(1); + const run = await scope.searches.run({ search: "mine" }); + expect(run.items.length).toBeGreaterThan(0); + await scope.searches.delete({ search: "mine" }); + expect((await scope.searches.list()).total).toBe(0); + }); + + it("packs, tasks, webhooks, analytics", async () => { + expect((await scope.packs.list()).items).toEqual([]); + await expect( + scope.registries.search({ query: "x", registry: "no-such-registry-for-tests" }), + ).rejects.toBeInstanceOf(NotFoundError); + expect(Array.isArray((await scope.tasks.list()).items)).toBe(true); + await expect(scope.tasks.get({ taskId: "missing" })).rejects.toBeInstanceOf(NotFoundError); + expect((await scope.webhooks.list()).items).toEqual([]); + expect((await scope.analytics.popular()).items).toEqual([]); + expect(await scope.analytics.searches()).toHaveProperty("knowledgeGaps"); + }); + + it("connectors are loaded on first use", async () => { + expect((await scope.connectors.list()).items).toEqual([]); + expect((await scope.connectors.sync({ all: true })).items).toEqual([]); + await expect(scope.connectors.sync({})).rejects.toBeInstanceOf(ValidationError); + }); + + it("admin: dedupe, pruneExpired, bulk ops, backup and restore", async () => { + await addDoc(scope, "Admin Doc"); + expect(Array.isArray((await scope.admin.dedupe()).items)).toBe(true); + expect(await scope.admin.pruneExpired()).toMatchObject({ pruned: 0 }); + expect((await scope.admin.bulkDelete({ library: "none", dryRun: true })).affected).toBe(0); + expect( + (await scope.admin.bulkRetag({ dateFrom: "2000-01-01", addTags: ["t"], dryRun: true })) + .affected, + ).toBe(1); + await scope.topics.create({ name: "target" }); + expect( + (await scope.admin.bulkMove({ dateFrom: "2000-01-01", targetTopic: "target", dryRun: true })) + .affected, + ).toBe(1); + + const file = join(tempHome, "backup.json"); + const backup = await scope.admin.backup({ outputPath: file }); + expect(backup.counts.documents).toBe(1); + const restored = await scope.admin.restore({ backupPath: file }); + expect(restored.path).toBe(file); + + const reindexed = await scope.admin.reindex({}, { onProgress: () => undefined }); + expect(reindexed.failed).toBe(0); + }); + + it("passes the per-call signal to the operation", async () => { + const controller = new AbortController(); + controller.abort(); + await expect(scope.docs.list({}, { signal: controller.signal })).rejects.toThrow(); + }); +}); + +describe("LibScope ask", () => { + it("throws a ConfigError with a hint when no LLM is configured", async () => { + const scope = create(); + try { + await addDoc(scope, "Ask Doc"); + await expect(scope.ask("What is this?")).rejects.toBeInstanceOf(ConfigError); + await expect(scope.askStream("What is this?").next()).rejects.toBeInstanceOf(ConfigError); + } finally { + scope.close(); + } + }); + + it("answers with the given LLM provider", async () => { + const { llm, complete } = mockLlm(); + const scope = create({ llmProvider: llm }); + try { + await addDoc(scope, "Ask Doc"); + const result = await scope.ask({ + question: "What is tested?", + topK: 2, + systemPrompt: "Be brief.", + }); + expect(result).toMatchObject({ mode: "answer", answer: "Mocked answer", model: "mock-llm" }); + expect(complete).toHaveBeenCalledOnce(); + expect(complete.mock.calls[0]?.[1]).toBe("Be brief."); + } finally { + scope.close(); + } + }); + + it("returns context in passthrough mode", async () => { + const scope = create({ config: { llm: { provider: "passthrough" } } }); + try { + await addDoc(scope, "Ask Doc"); + const result = await scope.ask("What is tested?"); + expect(result.mode).toBe("context"); + if (result.mode === "context") expect(result.contextPrompt).toContain("What is tested?"); + } finally { + scope.close(); + } + }); + + it("streams tokens and then sources", async () => { + const llm: LlmProvider = { + model: "stream-llm", + complete: vi.fn(), + async *completeStream() { + await Promise.resolve(); + yield "Hello"; + yield " world"; + }, + }; + const scope = create({ llmProvider: llm }); + try { + await addDoc(scope, "Stream Doc"); + const events = []; + for await (const event of scope.askStream("Question?")) events.push(event); + expect(events.slice(0, 2)).toEqual([{ token: "Hello" }, { token: " world" }]); + expect(events[2]).toMatchObject({ done: true, model: "stream-llm" }); + } finally { + scope.close(); + } + }); +}); + +describe("LibScope options", () => { + it("uses a custom chunker for added content", async () => { + const chunker = vi.fn().mockReturnValue(["part one", "part two", "part three"]); + const scope = create({ chunker }); + try { + const result = await scope.add({ content: "whole text", title: "Chunked" }); + expect(result.documents[0]!.chunkCount).toBe(3); + expect(chunker).toHaveBeenCalledWith({ content: "whole text", title: "Chunked", source: "" }); + + const file = join(tempHome, "notes.md"); + writeFileSync(file, "# Notes\n\nSome notes."); + const fromFile = await scope.add(file); + expect(fromFile.documents[0]!.chunkCount).toBe(3); + expect(chunker).toHaveBeenLastCalledWith( + expect.objectContaining({ title: "notes", source: file }), + ); + } finally { + scope.close(); + } + }); + + it("indexes a file with no parser (source code) when the chunker returns chunks", async () => { + const chunker = vi.fn(({ source }: { source: string }) => + source.endsWith(".ts") ? ["function a() {}", "function b() {}"] : undefined, + ); + const scope = create({ chunker }); + try { + const code = join(tempHome, "util.ts"); + writeFileSync(code, "function a() {}\nfunction b() {}\n"); + const result = await scope.add(code); + expect(result.documents[0]!.chunkCount).toBe(2); + + const binary = join(tempHome, "image.png"); + writeFileSync(binary, "not really a png"); + await expect(scope.add(binary)).rejects.toThrow(/Unsupported file format/); + } finally { + scope.close(); + } + }); + + it("leaves an injected database open on close", async () => { + const db = createTestDb(); + const scope = LibScope.create({ + db, + provider: new MockEmbeddingProvider(), + useConfigFile: false, + }); + await addDoc(scope, "Injected"); + scope.close(); + expect(db.open).toBe(true); + db.close(); + }); +}); diff --git a/tests/unit/lite-import-graph.test.ts b/tests/unit/lite-import-graph.test.ts new file mode 100644 index 00000000..06ae0020 --- /dev/null +++ b/tests/unit/lite-import-graph.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; + +/** + * `libscope/lite` (and the package root) must stay embeddable: their static import graph may + * not reach the CLI, MCP server, REST API, web UI or connectors. Connector operations are + * loaded with a dynamic import() the first time `scope.connectors.*` is called. + */ +const SRC = resolve(__dirname, "../../src"); +const FORBIDDEN_DIRS = ["cli/", "mcp/", "api/", "web/", "connectors/"]; +const FORBIDDEN_PACKAGES = ["commander", "@modelcontextprotocol/sdk", "node-cron"]; + +const STATEMENT_STARTS = ["import ", "export {", "export *"]; + +/** The module specifier of one import/export statement, or undefined if it has none. */ +function specifierOf(stmt: string): string | undefined { + const FROM = ' from "'; + const at = stmt.lastIndexOf(FROM); + let start: number; + if (at !== -1) start = at + FROM.length; + else if (stmt.startsWith('import "')) start = 'import "'.length; + else return undefined; + return stmt.slice(start, stmt.indexOf('"', start)); +} + +/** Module specifiers of value (not type-only) static imports and re-exports. */ +function staticImports(source: string): string[] { + const specs: string[] = []; + let stmt: string | undefined; + for (const line of source.split("\n")) { + if (stmt === undefined) { + if (!STATEMENT_STARTS.some((s) => line.startsWith(s))) continue; + stmt = line; + } else { + stmt += ` ${line.trim()}`; + } + if (!line.trimEnd().endsWith(";")) continue; + const typeOnly = stmt.startsWith("import type ") || stmt.startsWith("export type "); + const spec = typeOnly ? undefined : specifierOf(stmt); + if (spec !== undefined) specs.push(spec); + stmt = undefined; + } + return specs; +} + +function importGraph(entry: string): { files: Set; packages: Set } { + const files = new Set(); + const packages = new Set(); + const pending = [entry]; + while (pending.length > 0) { + const file = pending.pop()!; + if (files.has(file)) continue; + files.add(file); + for (const spec of staticImports(readFileSync(file, "utf-8"))) { + if (spec.startsWith(".")) { + pending.push(join(dirname(file), spec).replace(/\.js$/, ".ts")); + } else { + packages.add(spec); + } + } + } + return { files, packages }; +} + +describe.each(["lite/index.ts", "core/index.ts"])("import graph of src/%s", (entry) => { + const { files, packages } = importGraph(join(SRC, entry)); + const rel = [...files].map((f) => relative(SRC, f).replaceAll("\\", "/")); + + it("does not reach the CLI, MCP, REST, web or connector modules", () => { + expect(rel.filter((f) => FORBIDDEN_DIRS.some((d) => f.startsWith(d)))).toEqual([]); + }); + + it("does not import commander, the MCP SDK or the scheduler", () => { + expect([...packages].filter((p) => FORBIDDEN_PACKAGES.includes(p))).toEqual([]); + }); + + it("includes the operation layer (sanity check of the walker)", () => { + expect(rel).toContain("core/operations/documents.ts"); + }); +}); + +describe("import graph walker", () => { + it("finds the connectors behind the full operation index", () => { + const { files } = importGraph(join(SRC, "core/operations/index.ts")); + expect(files).toContain(join(SRC, "connectors/registry.ts")); + }); +}); diff --git a/tests/unit/lite.test.ts b/tests/unit/lite.test.ts index 7e64b11b..74e8611b 100644 --- a/tests/unit/lite.test.ts +++ b/tests/unit/lite.test.ts @@ -1,348 +1,188 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { LibScopeLite } from "../../src/lite/index.js"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + ConfigError, + LibScope, + ValidationError, + createCodeChunker, + createLite, + normalizeRawInput, + type LlmProvider, +} from "../../src/lite/index.js"; import { TreeSitterChunker } from "../../src/lite/chunker-treesitter.js"; -import type { LlmProvider } from "../../src/core/rag.js"; +import { initLogger } from "../../src/logger.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; + +const corpus = [ + { title: "React Hooks", content: "useState and useEffect are the most common React hooks." }, + { title: "Vue Composition", content: "Vue 3 composition API uses the setup function." }, + { title: "Angular DI", content: "Angular uses the dependency injection pattern extensively." }, +]; -function* fakeStream(): Generator { - yield "Hello"; - yield " world"; +async function seed(scope: LibScope): Promise { + for (const doc of corpus) await scope.add(doc); } -describe("LibScopeLite", () => { - let lite: LibScopeLite; - let provider: MockEmbeddingProvider; +/** Search results without the per-database IDs and timestamps. */ +function comparable(items: Array>): Array> { + return items.map(({ documentId: _d, chunkId: _c, ...rest }) => rest); +} - beforeEach(() => { - provider = new MockEmbeddingProvider(); - lite = new LibScopeLite({ dbPath: ":memory:", provider }); - }); +beforeAll(() => { + initLogger("silent"); +}); + +describe("createLite", () => { + const open: LibScope[] = []; + const lite = (options: Parameters[0]): LibScope => { + const scope = createLite(options); + open.push(scope); + return scope; + }; afterEach(() => { - lite.close(); + for (const scope of open.splice(0)) scope.close(); + vi.restoreAllMocks(); }); - describe("constructor", () => { - it("should create an instance with in-memory DB", () => { - expect(lite).toBeInstanceOf(LibScopeLite); - }); - - it("should accept custom embedding provider", () => { - const custom = new MockEmbeddingProvider(); - const instance = new LibScopeLite({ dbPath: ":memory:", provider: custom }); - expect(instance).toBeInstanceOf(LibScopeLite); - instance.close(); - }); + it("returns a LibScope", () => { + expect(lite({ dbPath: ":memory:", provider: new MockEmbeddingProvider() })).toBeInstanceOf( + LibScope, + ); }); - describe("index()", () => { - it("should index a single document", async () => { - await lite.index([{ title: "Test Doc", content: "This is test content for indexing." }]); - - expect(provider.embedBatchCallCount).toBeGreaterThan(0); - }); - - it("should index multiple documents", async () => { - await lite.index([ - { title: "Doc A", content: "Content of document A about TypeScript." }, - { title: "Doc B", content: "Content of document B about Python." }, - ]); - - // Both docs should have been processed - expect(provider.embedBatchCallCount).toBeGreaterThanOrEqual(2); - }); - - it("should index with optional metadata fields", async () => { - await lite.index([ - { - title: "Library Doc", - content: "React documentation content here.", - library: "react", - sourceType: "library", - version: "18.0.0", - url: "https://react.dev", - }, - ]); - - // Should succeed without errors - expect(provider.embedBatchCallCount).toBeGreaterThan(0); - }); + it("needs dbPath or db", () => { + expect(() => createLite({ provider: new MockEmbeddingProvider() })).toThrow(ValidationError); }); - describe("indexBatch()", () => { - it("should index documents with concurrency control", async () => { - const docs = Array.from({ length: 5 }, (_, i) => ({ - title: `Batch Doc ${i}`, - content: `Batch content number ${i} with enough text to be meaningful.`, - })); - - await lite.indexBatch(docs, { concurrency: 2 }); - - expect(provider.embedBatchCallCount).toBe(5); - }); - - it("should handle empty array", async () => { - await lite.indexBatch([], { concurrency: 2 }); - expect(provider.embedBatchCallCount).toBe(0); - }); - - it("should handle concurrency of 1 (sequential)", async () => { - const docs = [ - { title: "A", content: "Content A for sequential test." }, - { title: "B", content: "Content B for sequential test." }, - ]; - - await lite.indexBatch(docs, { concurrency: 1 }); - expect(provider.embedBatchCallCount).toBe(2); - }); + it("gives the same results as LibScope.create with useConfigFile: false", async () => { + const preset = lite({ dbPath: ":memory:", provider: new MockEmbeddingProvider() }); + const full = LibScope.create({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + useConfigFile: false, + }); + open.push(full); + await seed(preset); + await seed(full); + + const a = await preset.search({ query: "React hooks", limit: 2 }); + const b = await full.search({ query: "React hooks", limit: 2 }); + expect(a.total).toBe(b.total); + expect(comparable(a.items as unknown as Array>)).toEqual( + comparable(b.items as unknown as Array>), + ); + expect(a.items[0]).toHaveProperty("documentId"); + + const listA = await preset.docs.list(); + const listB = await full.docs.list(); + expect(listA.items.map((d) => d.title)).toEqual(listB.items.map((d) => d.title)); }); - describe("search()", () => { - beforeEach(async () => { - await lite.index([ - { - title: "React Hooks", - content: "useState and useEffect are the most common React hooks.", - }, - { title: "Vue Composition", content: "Vue 3 composition API uses setup function." }, - { title: "Angular DI", content: "Angular uses dependency injection pattern extensively." }, - ]); - }); - - it("should return search results", async () => { - const results = await lite.search("React hooks"); - expect(results.length).toBeGreaterThan(0); - }); - - it("should return results with expected shape", async () => { - const results = await lite.search("React"); - const first = results[0]; - expect(first).toBeDefined(); - expect(first).toHaveProperty("docId"); - expect(first).toHaveProperty("chunkId"); - expect(first).toHaveProperty("title"); - expect(first).toHaveProperty("content"); - expect(first).toHaveProperty("score"); - expect(first).toHaveProperty("url"); - expect(typeof first?.score).toBe("number"); - }); - - it("should respect limit option", async () => { - const results = await lite.search("API", { limit: 1 }); - expect(results.length).toBeLessThanOrEqual(1); - }); + it("ask without an LLM throws ConfigError; with one returns a RagResult", async () => { + const noLlm = lite({ dbPath: ":memory:", provider: new MockEmbeddingProvider() }); + await seed(noLlm); + await expect(noLlm.ask("What are hooks?")).rejects.toBeInstanceOf(ConfigError); + + const complete = vi.fn().mockResolvedValue({ text: "Hooks are functions." }); + const llm: LlmProvider = { model: "test-model", complete }; + const withLlm = lite({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + llmProvider: llm, + }); + await seed(withLlm); + const result = await withLlm.ask("What are hooks?"); + expect(result).toMatchObject({ + mode: "answer", + answer: "Hooks are functions.", + model: "test-model", + }); + if (result.mode === "answer") expect(result.sources.length).toBeGreaterThan(0); + expect(String(complete.mock.calls[0]?.[0])).toContain("What are hooks?"); }); - describe("getContext()", () => { - beforeEach(async () => { - await lite.index([ - { - title: "Node.js Streams", - content: "Readable streams in Node.js are a fundamental pattern.", - }, - ]); - }); - - it("should return a context string", async () => { - const context = await lite.getContext("How do Node.js streams work?"); - expect(typeof context).toBe("string"); - expect(context.length).toBeGreaterThan(0); + it("ignores config files: llm.provider can be set through config", async () => { + const scope = lite({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + config: { llm: { provider: "passthrough" } }, }); + await seed(scope); + expect((await scope.ask("What are hooks?")).mode).toBe("context"); }); +}); - describe("ask()", () => { - it("should throw when no LlmProvider is configured", async () => { - await lite.index([{ title: "Test", content: "Some content for testing ask." }]); - await expect(lite.ask("What is this about?")).rejects.toThrow("No LlmProvider configured"); - }); - - it("should call LlmProvider.complete with context", async () => { - // Declare the spy separately so we never reference it as an object method - const completeSpy = vi.fn().mockResolvedValue({ text: "Mocked LLM response" }); - const mockLlm: LlmProvider = { model: "test-model", complete: completeSpy }; - - const liteWithLlm = new LibScopeLite({ - dbPath: ":memory:", - provider, - llmProvider: mockLlm, - }); - - await liteWithLlm.index([ - { title: "Test Doc", content: "Information about testing patterns." }, - ]); - - const answer = await liteWithLlm.ask("What are testing patterns?"); - expect(answer).toBe("Mocked LLM response"); - expect(completeSpy).toHaveBeenCalledOnce(); - expect(completeSpy.mock.calls[0]?.[0]).toContain("testing patterns"); - - liteWithLlm.close(); - }); - - it("should allow llmProvider override in ask() opts", async () => { - const mockLlm: LlmProvider = { - model: "override-model", - complete: vi.fn().mockResolvedValue({ text: "Override response" }), - }; - - await lite.index([{ title: "Test", content: "Some content for LLM." }]); - - const answer = await lite.ask("Question?", { llmProvider: mockLlm }); - expect(answer).toBe("Override response"); - }); +describe("createCodeChunker", () => { + afterEach(() => { + vi.restoreAllMocks(); }); - describe("askStream()", () => { - it("should throw when no LlmProvider is configured", async () => { - await lite.index([{ title: "Test", content: "Content here." }]); - const gen = lite.askStream("Question?"); - await expect(gen.next()).rejects.toThrow("No LlmProvider configured"); - }); - - it("should throw when LlmProvider does not support streaming", async () => { - const mockLlm: LlmProvider = { - model: "no-stream", - complete: vi.fn().mockResolvedValue({ text: "done" }), - // No completeStream method - }; - - const liteWithLlm = new LibScopeLite({ - dbPath: ":memory:", - provider, - llmProvider: mockLlm, - }); - - await liteWithLlm.index([{ title: "Test", content: "Content." }]); - - const gen = liteWithLlm.askStream("Question?"); - await expect(gen.next()).rejects.toThrow("does not support streaming"); - - liteWithLlm.close(); - }); - - it("should stream tokens from LlmProvider", async () => { - const mockLlm: LlmProvider = { - model: "stream-model", - complete: vi.fn().mockResolvedValue({ text: "done" }), - completeStream: vi.fn().mockReturnValue(fakeStream()), - }; - - const liteWithLlm = new LibScopeLite({ - dbPath: ":memory:", - provider, - llmProvider: mockLlm, - }); - - await liteWithLlm.index([{ title: "Test", content: "Test content." }]); - - const tokens: string[] = []; - for await (const token of liteWithLlm.askStream("Question?")) { - tokens.push(token); - } - - expect(tokens).toEqual(["Hello", " world"]); - liteWithLlm.close(); - }); + const chunks = [ + { content: "function foo() {}", startLine: 1, endLine: 1, nodeType: "function_declaration" }, + { content: "function bar() {}", startLine: 2, endLine: 2, nodeType: "function_declaration" }, + ]; + + it("chunks code by the extension of the title", async () => { + const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk").mockResolvedValue(chunks); + const chunker = createCodeChunker(); + const result = await chunker({ content: "src", title: "main.ts", source: "" }); + expect(result).toEqual(["function foo() {}", "function bar() {}"]); + expect(chunkSpy).toHaveBeenCalledWith("src", "ts"); }); - describe("rate()", () => { - it("should rate an indexed document", async () => { - await lite.index([{ title: "Rate Me", content: "Content to rate." }]); - - // Find the doc ID via search - const results = await lite.search("rate"); - expect(results.length).toBeGreaterThan(0); - const docId = results[0]?.docId; - expect(docId).toBeDefined(); - - // Should not throw - lite.rate(docId!, 5); - }); - - it("should throw for nonexistent document", () => { - expect(() => lite.rate("nonexistent-doc", 3)).toThrow(); - }); + it("uses a fixed language when given", async () => { + const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk").mockResolvedValue(chunks); + await createCodeChunker({ language: "python" })({ content: "x", title: "Doc", source: "" }); + expect(chunkSpy).toHaveBeenCalledWith("x", "python"); }); - describe("close()", () => { - it("should close the database without error", () => { - const instance = new LibScopeLite({ dbPath: ":memory:", provider }); - expect(() => instance.close()).not.toThrow(); - }); + it("returns undefined for prose, unsupported languages and parse failures", async () => { + vi.spyOn(TreeSitterChunker.prototype, "chunk").mockRejectedValue(new Error("not installed")); + const chunker = createCodeChunker(); + expect(await chunker({ content: "x", title: "Guide", source: "" })).toBeUndefined(); + expect(await chunker({ content: "x", title: "main.rb", source: "" })).toBeUndefined(); + expect(await chunker({ content: "x", title: "main.go", source: "" })).toBeUndefined(); }); - describe("index() with language/tree-sitter chunking", () => { - afterEach(() => { - vi.restoreAllMocks(); - }); - - it("calls TreeSitterChunker.chunk() when language is set and supported", async () => { - vi.spyOn(TreeSitterChunker.prototype, "supports").mockReturnValue(true); - const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk").mockResolvedValue([ - { - content: "function foo() {}", - startLine: 1, - endLine: 3, - nodeType: "function_declaration", - }, - { - content: "function bar() {}", - startLine: 5, - endLine: 7, - nodeType: "function_declaration", - }, - ]); - - await lite.index([ - { - title: "src/main.ts", - content: "function foo() {}\nfunction bar() {}", - language: "typescript", - }, - ]); - - expect(chunkSpy).toHaveBeenCalledWith("function foo() {}\nfunction bar() {}", "typescript"); - }); - - it("does not call chunk() when language is not set", async () => { - const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk"); - - await lite.index([{ title: "Doc", content: "Some content here." }]); + it("is used by add when passed to createLite", async () => { + vi.spyOn(TreeSitterChunker.prototype, "chunk").mockResolvedValue(chunks); + const scope = createLite({ + dbPath: ":memory:", + provider: new MockEmbeddingProvider(), + chunker: createCodeChunker(), + }); + try { + const result = await scope.add({ content: "function foo() {}", title: "main.ts" }); + expect(result.documents[0]!.chunkCount).toBe(2); + } finally { + scope.close(); + } + }); +}); - expect(chunkSpy).not.toHaveBeenCalled(); +describe("normalizeRawInput", () => { + it("passes text through", async () => { + expect(await normalizeRawInput({ type: "text", title: "T", content: "C" })).toEqual({ + title: "T", + content: "C", }); + }); - it("falls back silently when tree-sitter throws", async () => { - vi.spyOn(TreeSitterChunker.prototype, "supports").mockReturnValue(true); - vi.spyOn(TreeSitterChunker.prototype, "chunk").mockRejectedValue( - new Error("tree-sitter not installed"), - ); - - // Should not throw — fallback to text chunker - await expect( - lite.index([ - { - title: "src/main.go", - content: "package main\nfunc main() {}", - language: "go", - }, - ]), - ).resolves.toBeUndefined(); + it("reads a buffer with the parser for its extension, else as text", async () => { + const md = await normalizeRawInput({ + type: "buffer", + buffer: Buffer.from("# Heading\n\nBody"), + filename: "notes.md", }); + expect(md.title).toBe("notes"); + expect(md.content).toContain("Body"); - it("does not call chunk() when language is set but not supported", async () => { - vi.spyOn(TreeSitterChunker.prototype, "supports").mockReturnValue(false); - const chunkSpy = vi.spyOn(TreeSitterChunker.prototype, "chunk"); - - await lite.index([ - { - title: "src/main.rb", - content: "def hello; end", - language: "ruby", - }, - ]); - - expect(chunkSpy).not.toHaveBeenCalled(); + const code = await normalizeRawInput({ + type: "buffer", + buffer: Buffer.from("const x = 1;"), + filename: "x.ts", + title: "x.ts", }); + expect(code).toEqual({ title: "x.ts", content: "const x = 1;" }); }); }); diff --git a/tests/unit/logger.test.ts b/tests/unit/logger.test.ts index 2a76554f..d6de4e77 100644 --- a/tests/unit/logger.test.ts +++ b/tests/unit/logger.test.ts @@ -1,5 +1,32 @@ -import { describe, it, expect, beforeEach } from "vitest"; -import { initLogger, getLogger, createChildLogger, withCorrelationId } from "../../src/logger.js"; +import { describe, it, expect, beforeEach, vi } from "vitest"; + +// Record which file descriptor each logger writes to, while still building real pino loggers. +const destinationCalls = vi.hoisted(() => [] as unknown[]); + +interface PinoModuleLike { + default: ((...args: unknown[]) => unknown) & { destination: (opts: unknown) => unknown }; +} + +vi.mock("pino", async (importOriginal) => { + const actual = await importOriginal(); + const realPino = actual.default; + const destination = (opts: unknown): unknown => { + destinationCalls.push(opts); + return realPino.destination(opts); + }; + const wrapped = Object.assign((...args: unknown[]) => realPino(...args), realPino, { + destination, + }); + return { ...actual, default: wrapped }; +}); + +const { initLogger, getLogger, createChildLogger, withCorrelationId } = + await import("../../src/logger.js"); + +function lastDestFd(): unknown { + const last = destinationCalls.at(-1) as { dest?: unknown } | undefined; + return last?.dest; +} describe("logger", () => { beforeEach(() => { @@ -30,3 +57,27 @@ describe("logger", () => { expect(logger.level).toBe("silent"); }); }); + +describe("logger destination", () => { + it("module-level logger (used before initLogger, e.g. config warnings) writes to stderr", () => { + const first = destinationCalls[0] as { dest?: unknown } | undefined; + expect(first?.dest).toBe(2); + }); + + it("writes to stderr (fd 2) by default so stdout stays clean", () => { + initLogger("info"); + expect(lastDestFd()).toBe(2); + }); + + it("writes to stderr when explicitly requested (MCP stdio server)", () => { + initLogger("info", { destination: "stderr" }); + expect(lastDestFd()).toBe(2); + expect(getLogger().level).toBe("info"); + }); + + it("can still target stdout when requested", () => { + initLogger("warn", { destination: "stdout" }); + expect(lastDestFd()).toBe(1); + expect(getLogger().level).toBe("warn"); + }); +}); diff --git a/tests/unit/mcp-entry.test.ts b/tests/unit/mcp-entry.test.ts new file mode 100644 index 00000000..532ea4bb --- /dev/null +++ b/tests/unit/mcp-entry.test.ts @@ -0,0 +1,87 @@ +import { describe, it, expect, vi } from "vitest"; +import { initLogger } from "../../src/logger.js"; +import { makeContext } from "./operations/helpers.js"; + +const mocks = vi.hoisted(() => ({ + transports: [] as unknown[], + bootstrap: vi.fn(), + initLogger: vi.fn(), +})); + +// A stand-in stdio transport, so no test touches the real stdin/stdout. +vi.mock("@modelcontextprotocol/sdk/server/stdio.js", () => ({ + StdioServerTransport: class { + onclose?: () => void; + constructor() { + mocks.transports.push(this); + } + start(): Promise { + return Promise.resolve(); + } + send(): Promise { + return Promise.resolve(); + } + close(): Promise { + this.onclose?.(); + return Promise.resolve(); + } + }, +})); + +vi.mock("../../src/core/bootstrap.js", async (importOriginal) => { + const orig = await importOriginal(); + return { + ...orig, + bootstrap: (...args: Parameters): ReturnType => { + mocks.bootstrap(); + return orig.bootstrap(...args); + }, + }; +}); + +vi.mock("../../src/logger.js", async (importOriginal) => { + const orig = await importOriginal(); + return { + ...orig, + initLogger: (...args: Parameters): void => { + mocks.initLogger(...args); + orig.initLogger(...args); + }, + }; +}); + +initLogger("silent"); + +describe("libscope/mcp entry points", () => { + it("importing the server module starts nothing", async () => { + const sigint = process.listenerCount("SIGINT"); + const mod = await import("../../src/mcp/server.js"); + expect(typeof mod.createMcpServer).toBe("function"); + expect(typeof mod.runStdioServer).toBe("function"); + expect(mocks.transports).toHaveLength(0); + expect(mocks.bootstrap).not.toHaveBeenCalled(); + expect(process.listenerCount("SIGINT")).toBe(sigint); + }); + + it("runStdioServer logs to stderr and connects the stdio transport", async () => { + const { runStdioServer } = await import("../../src/mcp/server.js"); + const t = makeContext({ surface: "mcp", llm: undefined }); + const before = { SIGINT: process.listeners("SIGINT"), SIGTERM: process.listeners("SIGTERM") }; + const mcp = await runStdioServer({ ctx: t.ctx, logLevel: "silent", toolsets: [] }); + try { + expect(mocks.initLogger).toHaveBeenCalledWith("silent", { destination: "stderr" }); + expect(mocks.transports).toHaveLength(1); + expect(mocks.bootstrap).not.toHaveBeenCalled(); + expect(mcp.tools).toHaveLength(11); + expect(process.listenerCount("SIGINT")).toBe(before.SIGINT.length + 1); + } finally { + for (const signal of ["SIGINT", "SIGTERM"] as const) { + for (const listener of process.listeners(signal)) { + if (!before[signal].includes(listener)) process.removeListener(signal, listener); + } + } + await mcp.close(); + t.db.close(); + } + }); +}); diff --git a/tests/unit/mcp-format-paging.test.ts b/tests/unit/mcp-format-paging.test.ts new file mode 100644 index 00000000..12d5c3ea --- /dev/null +++ b/tests/unit/mcp-format-paging.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import type { DocumentView } from "../../src/core/document-view.js"; +import { formatDocumentView, formatIngestResult, formatRating } from "../../src/mcp/format.js"; + +function makeView(overrides: Partial): DocumentView { + return { + document: { + documentId: "doc-1", + title: "Title", + sourceType: "manual", + library: null, + version: null, + topicId: null, + url: null, + contentHash: null, + submittedBy: "manual", + createdAt: "c", + updatedAt: "u", + }, + content: "body", + contentLength: 12, + offset: 0, + nextOffset: null, + tags: [], + links: { outgoing: [], incoming: [] }, + ratings: { documentId: "doc-1", averageRating: 0, totalRatings: 0, corrections: 0 }, + ...overrides, + }; +} + +describe("formatDocumentView paging", () => { + it("shows the range and the next offset on a first page", () => { + const text = formatDocumentView(makeView({ nextOffset: 4 })); + expect(text).toContain("\n\n[characters 0-4 of 12; next page: offset 4]\n\nbody"); + }); + + it("shows the range without a next offset on the last page", () => { + const text = formatDocumentView(makeView({ offset: 8 })); + expect(text).toContain("\n\n[characters 8-12 of 12]\n\nbody"); + }); + + it("shows no range for a whole document", () => { + expect(formatDocumentView(makeView({}))).not.toContain("[characters"); + }); +}); + +describe("formatIngestResult lines", () => { + it("lists added documents without a dry-run header", () => { + const text = formatIngestResult({ + kind: "file", + documents: [{ documentId: "d1", title: "Doc", chunkCount: 1 }], + errors: [], + skipped: [], + }); + expect(text).toBe("Added 1 document(s).\n- Doc (documentId: d1, 1 chunks)"); + }); +}); + +describe("formatRating without a chunk", () => { + it("leaves out the chunkId", () => { + expect( + formatRating({ + id: "r1", + documentId: "d1", + chunkId: null, + rating: 4, + feedback: "good", + suggestedCorrection: null, + ratedBy: "model", + createdAt: "now", + }), + ).toBe("Rated 4/5 (documentId: d1) · feedback saved"); + }); +}); diff --git a/tests/unit/mcp-format.test.ts b/tests/unit/mcp-format.test.ts new file mode 100644 index 00000000..a9b794d8 --- /dev/null +++ b/tests/unit/mcp-format.test.ts @@ -0,0 +1,297 @@ +import { describe, it, expect } from "vitest"; +import type { SearchResult } from "../../src/core/search.js"; +import type { Task } from "../../src/core/tasks.js"; +import type { DocumentView } from "../../src/core/document-view.js"; +import { + formatAnswer, + formatDocumentList, + formatDocumentView, + formatIngestResult, + formatOverview, + formatPackList, + formatRating, + formatReindex, + formatSearchResults, + formatSyncResult, + formatTask, + formatTaskCancel, + formatTaskList, +} from "../../src/mcp/format.js"; + +function makeResult(overrides: Partial = {}): SearchResult { + return { + documentId: "doc-1", + chunkId: "chunk-1", + title: "Doc One", + content: "chunk body", + sourceType: "manual", + library: null, + version: null, + topicId: null, + url: null, + score: 0.5, + avgRating: null, + scoreExplanation: { method: "vector", rawScore: 0.5, boostFactors: [], details: "" }, + ...overrides, + }; +} + +function makeTask(overrides: Partial = {}): Task { + return { + id: "task-1", + type: "operation", + operation: "add", + status: "running", + createdAt: new Date("2026-01-01T00:00:00Z"), + ...overrides, + }; +} + +const page = ( + items: T[], + total = items.length, + offset = 0, +): { items: T[]; total: number; limit: number; offset: number } => ({ + items, + total, + limit: 10, + offset, +}); + +describe("MCP output formatting", () => { + it("search results carry documentId, chunkId, metadata and context", () => { + const text = formatSearchResults( + page( + [ + makeResult({ + library: "react", + version: "18", + url: "https://example.com", + avgRating: 4, + contextBefore: [{ chunkId: "c0", content: "before text", chunkIndex: 0 }], + contextAfter: [{ chunkId: "c2", content: "after text", chunkIndex: 2 }], + }), + makeResult({ documentId: "doc-2", chunkId: "chunk-2", title: "Second" }), + ], + 20, + 10, + ), + ); + expect(text).toMatch(/^Results 11-12 of 20 — next page: offset 12\n/); + expect(text).toContain("[11] Doc One · score 0.50 · rating 4.0/5"); + expect(text).toContain( + "documentId: doc-1 · chunkId: chunk-1 · react v18 · https://example.com", + ); + expect(text).toContain( + "[context before]\nbefore text\nchunk body\n[context after]\nafter text", + ); + expect(text).toContain("---\n[12] Second"); + expect(text).toContain("documentId: doc-2 · chunkId: chunk-2"); + }); + + it("an empty page past the end says so", () => { + expect(formatSearchResults(page([], 3, 10))).toBe("No more results (total 3)."); + expect(formatSearchResults(page([], 0))).toBe("No results found."); + }); + + it("search header offers the next page when more results exist", () => { + expect(formatSearchResults(page([makeResult()], 5))).toMatch( + /^Results 1-1 of 5 — next page: offset 1\n/, + ); + }); + + it("ask: passthrough context and LLM answers list sources with documentIds", () => { + const sources = [{ documentId: "doc-1", title: "Doc One", chunk: "x", score: 0.9 }]; + const context = formatAnswer({ mode: "context", contextPrompt: "PROMPT", sources }); + expect(context).toMatch(/^Passthrough mode: no LLM was called/); + expect(context).toContain("PROMPT\n\nSources:\n- Doc One (documentId: doc-1, score 0.90)"); + + const answer = formatAnswer({ mode: "answer", answer: "Yes.", sources: [], model: "m" }); + expect(answer).toBe("Yes.\n\n(model: m)"); + const withTokens = formatAnswer({ + mode: "answer", + answer: "Yes.", + sources, + model: "m", + tokensUsed: 12, + }); + expect(withTokens).toContain("(model: m, 12 tokens)"); + }); + + it("document view shows metadata, url, suggested corrections and the whole content", () => { + const view: DocumentView = { + document: { + documentId: "doc-1", + title: "Title", + sourceType: "library", + library: "lib", + version: null, + topicId: "topic-1", + url: "https://x.test", + contentHash: null, + submittedBy: "manual", + createdAt: "c", + updatedAt: "u", + }, + content: "body", + contentLength: 4, + offset: 0, + nextOffset: null, + tags: [], + links: { outgoing: [], incoming: [] }, + ratings: { documentId: "doc-1", averageRating: 2, totalRatings: 2, corrections: 1 }, + }; + const text = formatDocumentView(view); + expect(text).toContain("documentId: doc-1 · library · lib · topic: topic-1"); + expect(text).toContain("url: https://x.test"); + expect(text).toContain("rating: 2.0/5 (2 ratings, 1 suggested corrections)"); + expect(text).not.toContain("characters"); + expect(text.endsWith("\n\nbody")).toBe(true); + }); + + it("document list includes IDs and metadata", () => { + const text = formatDocumentList( + page([ + { + documentId: "doc-1", + title: "T", + sourceType: "library", + library: "lib", + version: "2", + topicId: null, + url: null, + contentHash: null, + submittedBy: "manual", + createdAt: "c", + updatedAt: "u", + contentLength: 3, + }, + ]), + ); + expect(text).toBe("Documents 1-1 of 1\n- T (documentId: doc-1) library · lib v2"); + }); + + it("overview lists topics with IDs and the configured model", () => { + const text = formatOverview({ + stats: { + totalDocuments: 2, + totalChunks: 3, + totalTopics: 1, + databaseSizeBytes: 0, + totalSearches: 0, + avgLatencyMs: 0, + }, + topics: [{ id: "t1", name: "Infra", description: null, parentId: null, documentCount: 2 }], + packs: [], + index: { + stored: {}, + configured: { provider: "mock", model: "m", dimensions: 4 }, + }, + health: { database: "ok", fts: "error", vectorSearch: false }, + }); + expect(text).toContain("keyword index error · vector search unavailable"); + expect(text).toContain("Index embeddings: unknown · configured: mock/m, 4 dimensions"); + expect(text).toContain("Topics:\n- Infra (topic: t1, 2 docs)"); + }); + + it("ingest results: dry run, crawl stats, skipped and errors", () => { + const text = formatIngestResult({ + kind: "url", + documents: [{ documentId: "d1", title: "Page", chunkCount: 2, source: "https://a.test" }], + errors: [{ source: "https://b.test", error: "404" }], + skipped: [{ source: "https://c.test", reason: "unchanged" }], + planned: ["https://a.test"], + crawl: { pagesCrawled: 3, pagesSkipped: 1, abortReason: "maxPages" }, + }); + expect(text).toContain("Dry run: 1 source(s) would be added.\n- https://a.test"); + expect(text).toContain("- Page (documentId: d1, 2 chunks) https://a.test"); + expect(text).toContain("Crawled 3 page(s), skipped 1; stopped early: maxPages"); + expect(text).toContain("Skipped 1:\n- https://c.test: unchanged"); + expect(text).toContain("Errors 1:\n- https://b.test: 404"); + }); + + it("rating of a chunk includes the chunkId", () => { + expect( + formatRating({ + id: "r1", + documentId: "d1", + chunkId: "c1", + rating: 2, + feedback: null, + suggestedCorrection: "fix", + ratedBy: "model", + createdAt: "now", + }), + ).toBe("Rated 2/5 (documentId: d1, chunkId: c1) · correction saved"); + }); + + it("task status shows progress, errors, and formats or passes through results", () => { + const failed = formatTask( + makeTask({ status: "failed", error: "boom", progress: { current: 1, total: 0 } }), + ); + expect(failed).toContain("taskId: task-1 · add · failed 1/?"); + expect(failed).toContain("\nerror: boom"); + + const done = makeTask({ status: "completed", result: '{"n":1}' }); + expect(formatTask(done)).toContain('result:\n{"n":1}'); + expect(formatTask(done, () => "formatted")).toContain("result:\nformatted"); + expect(formatTask(done, () => undefined)).toContain('result:\n{"n":1}'); + const broken = makeTask({ status: "completed", result: "not json" }); + expect(formatTask(broken, () => "formatted")).toContain("result:\nnot json"); + const legacy = makeTask({ type: "sync_connector", operation: undefined, status: "pending" }); + expect(formatTask(legacy)).toContain("sync_connector · pending"); + }); + + it("task cancel and list", () => { + expect(formatTaskCancel({ taskId: "t", cancelRequested: false, status: "completed" })).toBe( + "Task t is already completed; nothing to cancel.", + ); + expect(formatTaskList({ items: [] })).toBe("No tasks in the last hour."); + expect(formatTaskList({ items: [makeTask()] })).toMatch(/^- taskId: task-1 · add · running/); + }); + + it("sync results show failures and per-item errors", () => { + const text = formatSyncResult({ + items: [ + { name: "a", status: "failed", error: "offline" }, + { + name: "b", + type: "slack", + status: "completed", + summary: { added: 1, updated: 2, deleted: 0, errors: ["#general: denied"] }, + }, + ], + }); + expect(text).toBe( + "- a: failed: offline\n- b (slack): 1 added, 2 updated, 0 deleted, 1 errors:\n #general: denied", + ); + }); + + it("pack list formats registry packs", () => { + const pack = { + name: "p", + description: "desc", + tags: [], + latestVersion: "1", + author: "a", + updatedAt: "2026-01-01", + registry: "team", + }; + expect(formatPackList({ items: [pack], warnings: ['Registry "x" was never synced'] })).toBe( + '- p v1 (registry team): desc\nWarning: Registry "x" was never synced', + ); + }); + + it("reindex reports failures and a rebuilt index", () => { + const text = formatReindex({ + total: 3, + completed: 2, + failed: 1, + failedChunkIds: ["c9"], + rebuiltIndex: "mock/m, 4 dimensions", + }); + expect(text).toBe( + "Re-embedded 2 of 3 chunks; 1 failed.\nRebuilt vector index: mock/m, 4 dimensions\nFailed chunkIds: c9", + ); + }); +}); diff --git a/tests/unit/mcp-server.test.ts b/tests/unit/mcp-server.test.ts index 1e0f65a8..c697b74d 100644 --- a/tests/unit/mcp-server.test.ts +++ b/tests/unit/mcp-server.test.ts @@ -1,209 +1,627 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { createTestDbWithVec } from "../fixtures/test-db.js"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import type { McpToolsetSetting } from "../../src/config.js"; +import type { LlmProvider } from "../../src/core/rag.js"; +import { createDatabase } from "../../src/db/connection.js"; +import { createVectorTable, runMigrations } from "../../src/db/schema.js"; +import { LibScopeError, ValidationError, DocumentNotFoundError } from "../../src/errors.js"; import { initLogger } from "../../src/logger.js"; import { errorResponse, withErrorHandling, type ToolResult } from "../../src/mcp/errors.js"; -import { LibScopeError, ValidationError, DocumentNotFoundError } from "../../src/errors.js"; -import type Database from "better-sqlite3"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { addDoc, makeContext, testConfig, type TestContext } from "./operations/helpers.js"; + +// Saved connector settings live under a temp HOME, never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-mcp-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const { createMcpServer, resolveToolsets, buildInstructions } = + await import("../../src/mcp/server.js"); +const { saveConnectorSettings } = await import("../../src/connectors/saved-config.js"); +const { taskRegistry } = await import("../../src/core/tasks.js"); +const { saveRegistries } = await import("../../src/registry/config.js"); +const { getPackDataPath, getRegistryCacheDir } = await import("../../src/registry/types.js"); + +initLogger("silent"); + +const CORE_TOOLS = [ + "search", + "ask", + "get-document", + "list-documents", + "overview", + "submit-document", + "update-document", + "delete-document", + "rate-document", + "link-documents", + "task", +]; +const ADMIN_TOOLS = ["sync", "install-pack", "list-packs", "reindex-documents"]; + +const fakeLlm: LlmProvider = { + model: "fake-model", + complete: () => Promise.resolve({ text: "The answer is 42.", tokensUsed: 9 }), +}; + +interface CallResult { + isError?: boolean; + content: Array<{ type: string; text?: string }>; +} + +/** + * MCP context where `ask` is passthrough (llm.provider "auto" on the MCP surface), on a + * database with a real sqlite-vec table so `search {relatedTo}` works. + */ +function passthroughContext(): TestContext { + const db = createDatabase(":memory:"); + runMigrations(db); + createVectorTable(db, new MockEmbeddingProvider()); + return makeContext({ db, surface: "mcp", llm: undefined }); +} + +async function connect( + t: TestContext, + toolsets: McpToolsetSetting[] = [], +): Promise<{ client: Client; close: () => Promise }> { + const mcp = createMcpServer({ ctx: t.ctx, toolsets }); + const client = new Client({ name: "test-client", version: "1.0.0" }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await Promise.all([mcp.server.connect(serverTransport), client.connect(clientTransport)]); + return { + client, + close: async (): Promise => { + await client.close(); + await mcp.close(); + }, + }; +} + +async function call( + client: Client, + name: string, + args: Record = {}, +): Promise<{ isError: boolean; text: string }> { + const result = (await client.callTool({ name, arguments: args })) as CallResult; + const text = result.content.map((c) => c.text ?? "").join("\n"); + return { isError: result.isError === true, text }; +} + +async function callOk( + client: Client, + name: string, + args: Record = {}, +): Promise { + const result = await call(client, name, args); + expect(result.isError, result.text).toBe(false); + return result.text; +} + +async function callError( + client: Client, + name: string, + args: Record = {}, +): Promise { + const result = await call(client, name, args); + expect(result.isError, result.text).toBe(true); + return result.text; +} + +/** documentId from a submit-document result. */ +function idFrom(text: string, key = "documentId"): string { + const match = new RegExp(`${key}: ([\\w-]+)`).exec(text); + if (!match?.[1]) throw new Error(`no ${key} in: ${text}`); + return match[1]; +} + +async function waitForTask(taskId: string): Promise { + for (let i = 0; i < 200; i++) { + const status = taskRegistry.get(taskId)?.status; + if (status && status !== "pending" && status !== "running") return; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error(`task ${taskId} did not finish`); +} + +describe("MCP error helpers", () => { + it("errorResponse formats LibScopeError, Error and other values", () => { + expect(errorResponse(new ValidationError("invalid input")).content[0]!.text).toBe( + "Error: invalid input", + ); + expect(errorResponse(new TypeError("bad type")).content[0]!.text).toBe( + "Error: TypeError: bad type", + ); + expect(errorResponse("raw").content[0]!.text).toContain("raw"); + expect(errorResponse(new LibScopeError("base")).isError).toBe(true); + expect(() => errorResponse(undefined)).not.toThrow(); + }); + + it("withErrorHandling passes results through and converts throws and rejections", async () => { + const ok: ToolResult = { content: [{ type: "text", text: "ok" }] }; + expect(await withErrorHandling(() => ok)({})).toEqual(ok); + const thrown = await withErrorHandling(() => { + throw new ValidationError("bad input"); + })({}); + expect(thrown.isError).toBe(true); + const rejected = await withErrorHandling(() => + Promise.reject(new DocumentNotFoundError("doc-123")), + )({}); + expect(rejected.content[0]!.text).toContain("doc-123"); + }); +}); + +describe("MCP server: tool list", () => { + let t: TestContext; -describe("MCP server helpers", () => { beforeEach(() => { - initLogger("silent"); + t = passthroughContext(); }); - describe("errorResponse", () => { - it("returns isError: true with text content", () => { - const result = errorResponse(new Error("something went wrong")); - expect(result.isError).toBe(true); - expect(result.content).toHaveLength(1); - expect(result.content[0]!.type).toBe("text"); - }); + afterEach(() => { + t.db.close(); + vi.unstubAllEnvs(); + }); - it("formats LibScopeError using just the message", () => { - const result = errorResponse(new ValidationError("invalid input")); - expect(result.content[0]!.text).toBe("Error: invalid input"); + it("registers the 11 core tools with annotations", async () => { + const { client, close } = await connect(t); + const { tools } = await client.listTools(); + expect(tools.map((tool) => tool.name)).toEqual(CORE_TOOLS); + + const byName = new Map(tools.map((tool) => [tool.name, tool])); + for (const name of ["search", "ask", "get-document", "list-documents", "overview"]) { + expect(byName.get(name)?.annotations?.readOnlyHint, name).toBe(true); + } + expect(byName.get("delete-document")?.annotations).toMatchObject({ + readOnlyHint: false, + destructiveHint: true, }); - - it("formats a generic Error using name: message", () => { - const err = new TypeError("bad type"); - const result = errorResponse(err); - expect(result.content[0]!.text).toBe("Error: TypeError: bad type"); + expect(byName.get("update-document")?.annotations).toMatchObject({ + destructiveHint: false, + idempotentHint: true, }); - - it("formats non-Error values using String()", () => { - const result = errorResponse("raw string error"); - expect(result.content[0]!.text).toContain("raw string error"); + expect(byName.get("submit-document")?.annotations).toMatchObject({ + readOnlyHint: false, + destructiveHint: false, }); - - it("formats null/undefined without throwing", () => { - expect(() => errorResponse(null)).not.toThrow(); - expect(() => errorResponse(undefined)).not.toThrow(); + expect(byName.get("link-documents")?.annotations).toMatchObject({ destructiveHint: true }); + expect(byName.get("task")?.annotations).toMatchObject({ + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, }); + for (const tool of tools) expect(tool.description, tool.name).toBeTruthy(); + await close(); }); - describe("withErrorHandling", () => { - it("returns the handler result when no error is thrown", async () => { - const expected: ToolResult = { content: [{ type: "text", text: "ok" }] }; - const wrapped = withErrorHandling(() => expected); - const result = await wrapped({}); - expect(result).toEqual(expected); - }); + it("generates input schemas from the operations", async () => { + const { client, close } = await connect(t); + const { tools } = await client.listTools(); + const props = (name: string): Record => + (tools.find((tool) => tool.name === name)?.inputSchema.properties ?? {}) as Record< + string, + { enum?: string[] } + >; + + expect(Object.keys(props("search"))).toEqual( + expect.arrayContaining(["query", "relatedTo", "topic", "library", "version", "sourceType"]), + ); + expect(Object.keys(props("search"))).toEqual( + expect.arrayContaining(["tags", "minRating", "limit", "offset"]), + ); + const submit = Object.keys(props("submit-document")); + expect(submit).toEqual(expect.arrayContaining(["content", "title", "url", "spider", "async"])); + for (const local of ["source", "kind", "format", "include", "exclude", "token"]) { + expect(submit).not.toContain(local); + } + expect(Object.keys(props("update-document"))).toContain("tags"); + expect(Object.keys(props("get-document"))).toEqual( + expect.arrayContaining(["documentId", "offset", "maxLength"]), + ); + expect(props("link-documents")["action"]?.enum).toEqual(["create", "delete"]); + expect(Object.keys(props("link-documents"))).toEqual( + expect.arrayContaining(["documentId", "targetDocumentId", "linkType", "linkId"]), + ); + expect(props("task")["action"]?.enum).toEqual(["status", "cancel", "list"]); + expect(Object.keys(props("search"))).not.toContain("async"); + await close(); + }); - it("catches synchronous throws and returns an error response", async () => { - const wrapped = withErrorHandling(() => { - throw new ValidationError("bad input"); - }); - const result = await wrapped({}); - expect(result.isError).toBe(true); - expect(result.content[0]!.text).toContain("bad input"); - }); + it("adds the admin toolset only when enabled", async () => { + const settings: McpToolsetSetting[][] = [["admin"], ["all"], ["core", "admin"]]; + for (const toolsets of settings) { + const { client, close } = await connect(t, toolsets); + const { tools } = await client.listTools(); + expect(tools.map((tool) => tool.name)).toEqual([...CORE_TOOLS, ...ADMIN_TOOLS]); + const byName = new Map(tools.map((tool) => [tool.name, tool])); + expect(byName.get("list-packs")?.annotations?.readOnlyHint).toBe(true); + expect(byName.get("sync")?.inputSchema.properties).not.toHaveProperty("token"); + await close(); + } + }); - it("catches rejected promises and returns an error response", async () => { - const wrapped = withErrorHandling(() => { - return Promise.reject(new DocumentNotFoundError("doc-123")); - }); - const result = await wrapped({}); - expect(result.isError).toBe(true); + it("reads mcp.toolsets from config when no toolsets are passed", () => { + const withAdmin = makeContext({ + db: t.db, + surface: "mcp", + llm: undefined, + config: { ...testConfig(), mcp: { toolsets: ["admin"] } }, }); + expect(createMcpServer({ ctx: withAdmin.ctx }).tools).toEqual([...CORE_TOOLS, ...ADMIN_TOOLS]); + expect(createMcpServer({ ctx: t.ctx }).tools).toEqual(CORE_TOOLS); + }); - it("passes params to the inner handler", async () => { - const handler = vi.fn().mockReturnValue({ content: [{ type: "text", text: "done" }] }); - const wrapped = withErrorHandling(handler); - const params = { docId: "abc", query: "test" }; - await wrapped(params); - expect(handler).toHaveBeenCalledWith(params); - }); + it("resolveToolsets expands 'all' and ignores 'core'", () => { + expect([...resolveToolsets(["admin"])]).toEqual(["admin"]); + expect([...resolveToolsets(["all"])]).toEqual(["admin"]); + expect([...resolveToolsets(["core"])]).toEqual([]); + expect([...resolveToolsets([])]).toEqual([]); + }); - it("returns isError: true for LibScopeError subclasses", async () => { - const wrapped = withErrorHandling(() => { - throw new LibScopeError("base lib error"); - }); - const result = await wrapped({}); - expect(result.isError).toBe(true); - expect(result.content[0]!.text).toBe("Error: base lib error"); - }); + it("omits ask when no LLM is configured and passthrough is off", async () => { + const noLlm = makeContext({ db: t.db, surface: "mcp", llm: null }); + const { client, close } = await connect(noLlm); + const names = (await client.listTools()).tools.map((tool) => tool.name); + expect(names).not.toContain("ask"); + expect(names).toHaveLength(10); + expect(client.getInstructions()).not.toContain("ask {question}"); + await close(); }); -}); -// Integration-style tests for MCP tool behaviors using the underlying core functions -// These verify the business logic that MCP tools delegate to. -describe("MCP tool business logic", () => { - let db: Database.Database; - let provider: MockEmbeddingProvider; + it("registers ask with a configured LLM and returns its answer", async () => { + await addDoc(t, "Answers", "The answer to everything is documented here"); + const withLlm = makeContext({ db: t.db, surface: "mcp", llm: fakeLlm }); + const { client, close } = await connect(withLlm); + const text = await callOk(client, "ask", { question: "What is the answer?" }); + expect(text).toContain("The answer is 42."); + expect(text).toContain("model: fake-model"); + expect(client.getInstructions()).toContain("configured LLM"); + await close(); + }); - beforeEach(() => { - initLogger("silent"); - db = createTestDbWithVec(); - provider = new MockEmbeddingProvider(); + it("sends instructions and the package version", async () => { + const { client, close } = await connect(t, ["admin"]); + const pkg = JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8")) as { + version: string; + }; + expect(client.getServerVersion()).toEqual({ name: "libscope", version: pkg.version }); + const instructions = client.getInstructions() ?? ""; + expect(instructions).toContain("search {query}"); + expect(instructions).toContain("get-document {documentId}"); + expect(instructions).toContain("rate-document"); + expect(instructions).toContain("without calling an LLM"); + expect(instructions).toContain("Admin tools"); + await close(); }); - afterEach(() => { - db.close(); + it("buildInstructions leaves out ask and admin when unavailable", () => { + const text = buildInstructions({ ask: false, passthrough: false, admin: false }); + expect(text).not.toContain("ask {question}"); + expect(text).not.toContain("Admin tools"); }); +}); + +describe("MCP server: tool calls", () => { + let t: TestContext; + let client: Client; + let close: () => Promise; - it("search returns empty response when no documents are indexed", async () => { - const { searchDocuments } = await import("../../src/core/search.js"); - const { results, totalCount } = await searchDocuments(db, provider, { query: "anything" }); - expect(results).toHaveLength(0); - expect(totalCount).toBe(0); + beforeEach(async () => { + tempHome = join(tmpdir(), `libscope-mcp-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + t = passthroughContext(); + ({ client, close } = await connect(t, ["admin"])); }); - it("indexDocument then getDocument returns indexed document", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { getDocument } = await import("../../src/core/documents.js"); + afterEach(async () => { + await close(); + t.db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); - const indexed = await indexDocument(db, provider, { - title: "Test Doc", - content: "Some content for testing.", - sourceType: "manual", - }); + async function submit(title: string, content: string): Promise { + return idFrom(await callOk(client, "submit-document", { title, content })); + } + + it("search: finds by query and relatedTo, with IDs; rejects bad input", async () => { + const id = await submit("Kubernetes", "Kubernetes pods and deployments explained"); + await submit("Pods", "More about kubernetes pods"); + + const text = await callOk(client, "search", { query: "kubernetes pods" }); + expect(text).toMatch(/^Results 1-2 of 2/); + expect(text).toContain(`documentId: ${id}`); + expect(text).toContain("chunkId: "); - expect(indexed.id).toBeTruthy(); + const related = await callOk(client, "search", { relatedTo: id }); + expect(related).not.toContain(`documentId: ${id}`); - const fetched = getDocument(db, indexed.id); - expect(fetched.title).toBe("Test Doc"); - expect(fetched.content).toBe("Some content for testing."); + expect(await callOk(client, "search", { query: "kubernetes", library: "none" })).toBe( + "No results found.", + ); + expect(await callError(client, "search", {})).toContain("Give either query or relatedTo"); + expect(await callError(client, "search", { query: 5 })).toMatch(/query/); + expect(await callError(client, "search", { query: "x", limit: 0 })).toMatch(/limit/); }); - it("deleteDocument removes a document", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { deleteDocument, getDocument } = await import("../../src/core/documents.js"); + it("ask: passthrough returns the retrieved context; rejects a missing question", async () => { + const id = await submit("Deploys", "Deploy with kubectl apply"); + const text = await callOk(client, "ask", { question: "How do I deploy?" }); + expect(text).toContain("Passthrough mode"); + expect(text).toContain("How do I deploy?"); + expect(text).toContain(`documentId: ${id}`); + expect(await callError(client, "ask", {})).toMatch(/question/); + }); - const indexed = await indexDocument(db, provider, { - title: "Delete Me", - content: "This will be deleted.", - sourceType: "manual", + it("get-document: returns metadata, tags, links and paged content", async () => { + const id = await submit("Guide", "abcdefghij".repeat(5)); + const other = await submit("Other", "Other content"); + const linkText = await callOk(client, "link-documents", { + action: "create", + documentId: id, + targetDocumentId: other, + linkType: "see_also", }); + const linkId = idFrom(linkText, "linkId"); + await callOk(client, "update-document", { documentId: id, tags: ["k8s"] }); + + const full = await callOk(client, "get-document", { documentId: id }); + expect(full).toContain("# Guide"); + expect(full).toContain(`documentId: ${id}`); + expect(full).toContain("tags: k8s"); + expect(full).toContain(`linkId: ${linkId}`); + + const page = await callOk(client, "get-document", { documentId: id, maxLength: 20 }); + expect(page).toContain("characters 0-20 of 50; next page: offset 20"); + const last = await callOk(client, "get-document", { documentId: id, offset: 40 }); + expect(last).toContain("characters 40-50 of 50]"); + + expect(await callError(client, "get-document", { documentId: "missing" })).toContain("missing"); + expect(await callError(client, "get-document", {})).toMatch(/documentId/); + }); - deleteDocument(db, indexed.id); - - expect(() => getDocument(db, indexed.id)).toThrow(DocumentNotFoundError); + it("list-documents: pages with offset and total", async () => { + const a = await submit("Doc A", "Content A"); + await submit("Doc B", "Content B"); + const first = await callOk(client, "list-documents", { limit: 1 }); + expect(first).toContain("Documents 1-1 of 2 — next page: offset 1"); + const second = await callOk(client, "list-documents", { limit: 1, offset: 1 }); + expect(second).toContain("Documents 2-2 of 2"); + expect(first + second).toContain(`documentId: ${a}`); + expect(await callError(client, "list-documents", { limit: 0 })).toMatch(/limit/); }); - it("listDocuments returns paginated documents", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { listDocuments } = await import("../../src/core/documents.js"); + it("overview: shows counts, health and index", async () => { + await submit("Doc", "Content"); + const text = await callOk(client, "overview"); + expect(text).toContain("Documents: 1"); + expect(text).toContain("Health: database ok"); + expect(text).toContain("Index embeddings:"); + }); - await indexDocument(db, provider, { - title: "Doc A", - content: "Content A", - sourceType: "library", - library: "react", + it("submit-document: content, async task, and invalid input", async () => { + const text = await callOk(client, "submit-document", { + title: "Inline", + content: "Inline content", + tags: ["a"], }); - await indexDocument(db, provider, { - title: "Doc B", - content: "Content B", - sourceType: "library", - library: "vue", + expect(text).toContain("Added 1 document(s)."); + expect(text).toMatch(/documentId: [\w-]+, 1 chunks/); + + const started = await callOk(client, "submit-document", { + title: "Background", + content: "Background content", + async: true, }); + const taskId = idFrom(started, "taskId"); + await waitForTask(taskId); + const status = await callOk(client, "task", { action: "status", taskId }); + expect(status).toContain("completed"); + expect(status).toContain("Added 1 document(s)."); + expect(status).toContain("Background (documentId: "); + + expect(await callError(client, "submit-document", {})).toContain("Provide content"); + // Checked before a background task starts: the error comes back at once, with no taskId. + const background = await callError(client, "submit-document", { async: true }); + expect(background).toContain("Provide content"); + expect(background).not.toContain("taskId"); + expect(await callError(client, "submit-document", { url: "/etc/passwd" })).toMatch(/url/); + expect(await callError(client, "submit-document", { title: "T", content: "" })).toMatch( + /content/, + ); + }); - const all = listDocuments(db, {}); - expect(all.length).toBeGreaterThanOrEqual(2); + it("update-document: changes fields and tags; rejects empty updates", async () => { + const id = await submit("Old", "Old content"); + const text = await callOk(client, "update-document", { + documentId: id, + title: "New", + tags: ["x", "y"], + }); + expect(text).toContain("# New"); + expect(text).toContain("tags: x, y"); + expect(await callError(client, "update-document", { documentId: id })).toContain( + "Nothing to update", + ); + expect( + await callError(client, "update-document", { documentId: "missing", title: "x" }), + ).toContain("missing"); + }); - const limited = listDocuments(db, { limit: 1 }); - expect(limited).toHaveLength(1); + it("delete-document: deletes; errors for unknown IDs", async () => { + const id = await submit("Gone", "Soon gone"); + expect(await callOk(client, "delete-document", { documentId: id })).toBe( + `Deleted document ${id}.`, + ); + await callError(client, "get-document", { documentId: id }); + await callError(client, "delete-document", { documentId: id }); + await callError(client, "delete-document", {}); }); - it("getDocumentRatings returns zero ratings for new document", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { getDocumentRatings } = await import("../../src/core/ratings.js"); + it("rate-document: records a rating; rejects out-of-range ratings", async () => { + const id = await submit("Rated", "Rated content"); + const text = await callOk(client, "rate-document", { + documentId: id, + rating: 4, + feedback: "useful", + }); + expect(text).toContain(`Rated 4/5 (documentId: ${id})`); + expect(text).toContain("feedback saved"); + expect(await callOk(client, "get-document", { documentId: id })).toContain( + "rating: 4.0/5 (1 ratings)", + ); + await callError(client, "rate-document", { documentId: id, rating: 6 }); + await callError(client, "rate-document", { documentId: "missing", rating: 3 }); + }); - const indexed = await indexDocument(db, provider, { - title: "Rate Me", - content: "Rateable content.", - sourceType: "manual", + it("link-documents: creates and deletes links; validates per action", async () => { + const a = await submit("A", "Doc A"); + const b = await submit("B", "Doc B"); + const created = await callOk(client, "link-documents", { + action: "create", + documentId: a, + targetDocumentId: b, + linkType: "prerequisite", + label: "read first", }); + expect(created).toContain(`Linked ${a} -> ${b} (prerequisite) "read first"`); + const linkId = idFrom(created, "linkId"); + expect(await callOk(client, "link-documents", { action: "delete", linkId })).toBe( + `Deleted link ${linkId}.`, + ); + + expect(await callError(client, "link-documents", { action: "create", documentId: a })).toMatch( + /targetDocumentId/, + ); + expect(await callError(client, "link-documents", { action: "delete" })).toMatch(/linkId/); + await callError(client, "link-documents", { action: "rename", linkId }); + await callError(client, "link-documents", { action: "delete", linkId: "missing" }); + }); - const ratings = getDocumentRatings(db, indexed.id); - expect(ratings.totalRatings).toBe(0); - expect(ratings.averageRating).toBe(0); + it("task: status, cancel and list; errors for unknown tasks", async () => { + const { task } = taskRegistry.run( + "operation", + (signal) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason as Error)); + }), + "add", + ); + const running = await callOk(client, "task", { action: "status", taskId: task.id }); + expect(running).toContain(`taskId: ${task.id}`); + expect(running).toContain("running"); + expect(await callOk(client, "task", { action: "list" })).toContain(task.id); + + const cancelled = await callOk(client, "task", { action: "cancel", taskId: task.id }); + expect(cancelled).toContain(`Cancellation requested for task ${task.id}`); + await waitForTask(task.id); + expect(await callOk(client, "task", { action: "cancel", taskId: task.id })).toContain( + "already cancelled", + ); + + expect(await callError(client, "task", { action: "status", taskId: "nope" })).toContain( + "not found", + ); + await callError(client, "task", { action: "cancel", taskId: "nope" }); + await callError(client, "task", { action: "status" }); + await callError(client, "task", {}); }); - it("rateDocument stores a rating and updates average", async () => { - const { indexDocument } = await import("../../src/core/indexing.js"); - const { rateDocument, getDocumentRatings } = await import("../../src/core/ratings.js"); + it("sync (admin): syncs saved connections without secrets as parameters", async () => { + expect(await callOk(client, "sync", { all: true })).toBe("No saved connections to sync."); - const indexed = await indexDocument(db, provider, { - title: "Rate Me", - content: "Rateable content.", - sourceType: "manual", + const vault = join(tempHome, "vault"); + mkdirSync(vault, { recursive: true }); + writeFileSync(join(vault, "note.md"), "# Note\n\nA note in the vault."); + saveConnectorSettings("obsidian", "notes", { + vaultPath: vault, + topicMapping: "folder", + excludePatterns: [], }); + expect(await callOk(client, "sync", { name: "notes" })).toBe( + "- notes (obsidian): 1 added, 0 updated, 0 deleted", + ); + + expect(await callError(client, "sync", {})).toContain("Give either a connection name"); + await callError(client, "sync", { name: "missing" }); + await callError(client, "sync", { name: "bad name!" }); + }); - rateDocument(db, { documentId: indexed.id, rating: 4, feedback: "good doc" }); - const ratings = getDocumentRatings(db, indexed.id); - expect(ratings.totalRatings).toBe(1); - expect(ratings.averageRating).toBe(4); + it("install-pack and list-packs (admin): install from a registry and list", async () => { + // A synced registry "team" holding pack "demo" (local copy only, no git needed). + saveRegistries([{ name: "team", url: "https://example.com/team.git", lastSyncedAt: null }]); + const dataPath = getPackDataPath("team", "demo", "1.0.0"); + mkdirSync(join(dataPath, ".."), { recursive: true }); + const pack = { + name: "demo", + version: "1.0.0", + description: "Test pack", + documents: [{ title: "Pack doc", content: "Content from a pack", source: "test" }], + metadata: { author: "tests", license: "MIT", createdAt: new Date().toISOString() }, + }; + writeFileSync(dataPath, JSON.stringify(pack)); + const summary = { name: "demo", description: "Test pack", tags: [], latestVersion: "1.0.0" }; + writeFileSync( + join(getRegistryCacheDir("team"), "index.json"), + JSON.stringify([{ ...summary, author: "tests", updatedAt: "2026-01-01" }]), + ); + + expect(await callOk(client, "list-packs")).toBe("No packs found."); + expect(await callOk(client, "list-packs", { available: true })).toBe( + "- demo v1.0.0 (registry team): Test pack", + ); + expect(await callOk(client, "install-pack", { pack: "demo" })).toBe( + "Installed pack demo: 1 documents.", + ); + expect(await callOk(client, "install-pack", { pack: "demo@1.0.0", registry: "team" })).toBe( + "Pack demo is already installed.", + ); + expect(await callOk(client, "list-packs")).toMatch(/^- demo v1\.0\.0 \(1 docs, installed /); + expect(await callOk(client, "overview")).toContain("Packs:\n- demo v1.0.0"); + + // Local files are not read over MCP. + const file = join(tempHome, "demo.json"); + writeFileSync(file, JSON.stringify(pack)); + expect(await callError(client, "install-pack", { pack: file })).toContain( + "only available from the CLI", + ); + await callError(client, "install-pack", {}); + await callError(client, "install-pack", { pack: "missing" }); + await callError(client, "list-packs", { available: "yes" }); }); - it("listTopics returns empty array when no topics exist", async () => { - const { listTopics } = await import("../../src/core/topics.js"); - const topics = listTopics(db); - expect(topics).toEqual([]); + it("reindex-documents (admin): re-embeds chunks, inline or as a task", async () => { + await submit("Doc", "Reindex me"); + expect(await callOk(client, "reindex-documents")).toBe("Re-embedded 1 of 1 chunks."); + const started = await callOk(client, "reindex-documents", { async: true }); + const taskId = idFrom(started, "taskId"); + await waitForTask(taskId); + expect(await callOk(client, "task", { action: "status", taskId })).toContain( + "Re-embedded 1 of 1 chunks.", + ); + await callError(client, "reindex-documents", { batchSize: 0 }); + await callError(client, "reindex-documents", { rebuild: true, documentIds: ["x"] }); }); +}); - it("errorResponse for DocumentNotFoundError returns proper message", () => { - const result = errorResponse(new DocumentNotFoundError("missing-id")); - expect(result.isError).toBe(true); - expect(result.content[0]!.text).toContain("missing-id"); +describe("createMcpServer without a context", () => { + it("bootstraps its own database and closes it", async () => { + const mcp = createMcpServer({ + dbPath: ":memory:", + config: testConfig(), + provider: new MockEmbeddingProvider(), + vectorTable: "skip", + toolsets: [], + }); + expect(mcp.tools).toEqual(CORE_TOOLS); + expect(mcp.ctx.surface).toBe("mcp"); + expect(mcp.ctx.db.open).toBe(true); + await mcp.close(); + expect(mcp.ctx.db.open).toBe(false); }); }); diff --git a/tests/unit/obsidian-wikilinks.test.ts b/tests/unit/obsidian-wikilinks.test.ts new file mode 100644 index 00000000..097d50f0 --- /dev/null +++ b/tests/unit/obsidian-wikilinks.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { findWikilinks, replaceWikilinks } from "../../src/connectors/obsidian-wikilinks.js"; + +const links = (text: string): Array<[string, string | undefined]> => + findWikilinks(text).map((w) => [w.link, w.display]); + +describe("findWikilinks", () => { + it("finds plain links and links with display text, with their positions", () => { + expect(findWikilinks("see [[Note]] and [[Other Note|shown]].")).toEqual([ + { start: 4, end: 12, link: "Note", display: undefined }, + { start: 17, end: 37, link: "Other Note", display: "shown" }, + ]); + }); + + it.each([ + ["an embed", "![[image.png]]", []], + ["an empty link", "[[]]", []], + ["a link that starts with |", "[[|x]]", []], + ["an unclosed link", "[[Note", []], + ["a single ] at the end", "[[Note]", []], + ["an unclosed display", "[[a|b", []], + ["a display with a single ]", "[[a|b]c]]", []], + ])("ignores %s", (_case, text, expected) => { + expect(links(text)).toEqual(expected); + }); + + it.each([ + ["an empty display", "[[a|]]", [["a", ""]]], + ["a display with |", "[[a|b|c]]", [["a", "b|c"]]], + ["a display with [[", "[[a|b [[c]]", [["a", "b [[c"]]], + ["the innermost [[ of a run", "[[[[a]]", [["[[a", undefined]]], + ["a link after an embed", "![[e]] [[l]]", [["l", undefined]]], + ["a link after a failed one", "[[a] [[b]]", [["b", undefined]]], + ["a link after an unclosed display", "[[a|b] [[c]]", [["c", undefined]]], + ["a link that spans lines", "[[a\nb]]", [["a\nb", undefined]]], + ["a [[ right after [", "[[[a]]", [["[a", undefined]]], + ])("finds %s", (_case, text, expected) => { + expect(links(text)).toEqual(expected); + }); + + it("scans long runs of brackets and pipes quickly", () => { + const hostile = "[[".repeat(50_000) + "[[a|".repeat(50_000) + "[[a]".repeat(50_000); + const started = Date.now(); + expect(findWikilinks(hostile)).toEqual([]); + expect(Date.now() - started).toBeLessThan(1000); + }); +}); + +describe("replaceWikilinks", () => { + it("replaces each link and keeps the text around it", () => { + const out = replaceWikilinks("a [[x]] b [[y|Y]] ![[z]] c", (w) => `<${w.display ?? w.link}>`); + expect(out).toBe("a b ![[z]] c"); + }); + + it("returns the text unchanged when there are no links", () => { + expect(replaceWikilinks("no links [here]", () => "?")).toBe("no links [here]"); + }); +}); diff --git a/tests/unit/onenote-device-code.test.ts b/tests/unit/onenote-device-code.test.ts new file mode 100644 index 00000000..8b349717 --- /dev/null +++ b/tests/unit/onenote-device-code.test.ts @@ -0,0 +1,73 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { authenticateDeviceCode } from "../../src/connectors/onenote.js"; + +const sleep = vi.hoisted(() => vi.fn((_ms: number) => Promise.resolve())); +vi.mock("node:timers/promises", () => ({ setTimeout: sleep })); + +function json(data: unknown, status = 200): Response { + return new Response(JSON.stringify(data), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +/** Device code response, then the given token responses in order. */ +function stubSignIn(expiresIn: number, tokenResponses: Array<() => Response>): string[] { + const calls: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn((url: string) => { + calls.push(url.includes("/devicecode") ? "devicecode" : "token"); + if (url.includes("/devicecode")) { + return Promise.resolve( + json({ + device_code: "dev", + user_code: "CODE", + verification_uri: "https://example.test/login", + expires_in: expiresIn, + interval: 2, + }), + ); + } + const next = tokenResponses.shift(); + return Promise.resolve(next ? next() : json({ error: "authorization_pending" }, 400)); + }), + ); + return calls; +} + +describe("authenticateDeviceCode polling", () => { + beforeEach(() => { + sleep.mockClear(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("waits the extra 5 seconds after slow_down and keeps polling", async () => { + const calls = stubSignIn(900, [ + (): Response => json({ error: "slow_down" }, 400), + (): Response => json({ error: "authorization_pending" }, 400), + (): Response => json({ access_token: "a", refresh_token: "r", expires_in: 60 }), + ]); + + const result = await authenticateDeviceCode("client"); + + expect(result).toMatchObject({ accessToken: "a", refreshToken: "r" }); + expect(calls).toEqual(["devicecode", "token", "token", "token"]); + expect(sleep.mock.calls.map(([ms]) => ms)).toEqual([2000, 5000, 2000, 2000]); + }); + + it("times out without polling when the code has already expired", async () => { + const calls = stubSignIn(0, []); + + await expect(authenticateDeviceCode("client")).rejects.toThrow( + "Device code authentication timed out", + ); + expect(calls).toEqual(["devicecode"]); + expect(sleep).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/operations/connectors.test.ts b/tests/unit/operations/connectors.test.ts new file mode 100644 index 00000000..e29aa603 --- /dev/null +++ b/tests/unit/operations/connectors.test.ts @@ -0,0 +1,110 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import { NotFoundError, ValidationError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +// Saved connections live under a temp HOME, never the real ~/.libscope. +let tempHome = join(tmpdir(), `libscope-ops-connectors-${process.pid}`); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const ops = await import("../../../src/core/operations/index.js"); +const { saveConnectorSettings } = await import("../../../src/connectors/saved-config.js"); +const { hasNamedConnectorConfig } = await import("../../../src/connectors/index.js"); + +initLogger("silent"); + +describe("connector operations", () => { + let t: TestContext; + let vault: string; + + beforeEach(() => { + tempHome = join(tmpdir(), `libscope-ops-connectors-${randomUUID()}`); + vault = join(tempHome, "vault"); + mkdirSync(vault, { recursive: true }); + writeFileSync(join(vault, "note.md"), "# Note\n\nA note in the vault."); + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + rmSync(tempHome, { recursive: true, force: true }); + }); + + function saveVault(name: string): void { + saveConnectorSettings("obsidian", name, { + vaultPath: vault, + topicMapping: "folder", + excludePatterns: [], + }); + } + + it("lists, syncs by name, syncs all and disconnects a saved connection", async () => { + saveVault("notes"); + saveConnectorSettings("slack", "team-slack", { + token: "xoxb-secret", + channels: ["all"], + threadMode: "aggregate", + }); + + const listed = await run(ops.listConnectionsOperation, t.ctx, {}); + expect(listed.items.map((c) => c.name)).toEqual(["notes", "team-slack"]); + expect(listed.items[1]?.settings["token"]).toBe("***"); + + const one = await run(ops.syncOperation, t.ctx, { name: "notes" }); + expect(one.items[0]).toMatchObject({ + name: "notes", + type: "obsidian", + status: "completed", + summary: { added: 1 }, + }); + + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("offline"))); + try { + const all = await run(ops.syncOperation, t.ctx, { all: true }); + expect(all.items.map((i) => [i.name, i.status])).toEqual([ + ["notes", "completed"], + ["team-slack", "failed"], + ]); + } finally { + vi.unstubAllGlobals(); + } + + const afterSync = await run(ops.listConnectionsOperation, t.ctx, {}); + expect(afterSync.items[0]?.lastRun?.status).toBe("completed"); + + const removed = await run(ops.disconnectOperation, t.ctx, { name: "notes" }); + expect(removed).toMatchObject({ type: "obsidian", documentsRemoved: 1, configRemoved: true }); + expect(hasNamedConnectorConfig("notes")).toBe(false); + }); + + it("needs exactly one of name and all, and a safe name", async () => { + await expectValidationError(run(ops.syncOperation, t.ctx, {})); + await expectValidationError(run(ops.syncOperation, t.ctx, { name: "a", all: true })); + await expectValidationError(run(ops.disconnectOperation, t.ctx, { name: "../etc" })); + }); + + it("throws NotFoundError for an unknown connection", async () => { + await expect(run(ops.syncOperation, t.ctx, { name: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + }); + + it("disconnects by type when nothing is saved, and checks the type", async () => { + const result = await run(ops.disconnectOperation, t.ctx, { name: "slack" }); + expect(result).toMatchObject({ type: "slack", documentsRemoved: 0, configRemoved: false }); + await expect(run(ops.disconnectOperation, t.ctx, { name: "unknown" })).rejects.toBeInstanceOf( + NotFoundError, + ); + saveVault("notes"); + await expect( + run(ops.disconnectOperation, t.ctx, { name: "notes", type: "slack" }), + ).rejects.toBeInstanceOf(ValidationError); + }); +}); diff --git a/tests/unit/operations/documents.test.ts b/tests/unit/operations/documents.test.ts new file mode 100644 index 00000000..a6631837 --- /dev/null +++ b/tests/unit/operations/documents.test.ts @@ -0,0 +1,231 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + addOperation, + deleteDocumentOperation, + documentHistoryOperation, + getDocumentOperation, + listDocumentsOperation, + rateDocumentOperation, + rollbackDocumentOperation, + startOperationTask, + updateDocumentOperation, +} from "../../../src/core/operations/index.js"; +import { createTopic } from "../../../src/core/topics.js"; +import { saveVersion } from "../../../src/core/versioning.js"; +import { DocumentNotFoundError, NotFoundError, TopicNotFoundError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("document operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + describe("add", () => { + it("adds inline content with tags and the default source type", async () => { + createTopic(t.db, { name: "Guides" }); + const result = await run(addOperation, t.ctx, { + title: "Setup", + content: "How to set up the project", + topic: "guides", + tags: ["Setup"], + }); + expect(result.kind).toBe("content"); + expect(result.documents).toHaveLength(1); + const id = result.documents[0]!.documentId; + const view = await run(getDocumentOperation, t.ctx, { documentId: id }); + expect(view.document.sourceType).toBe("topic"); + expect(view.tags).toEqual(["setup"]); + }); + + it("rejects input without content or source", async () => { + await expectValidationError(run(addOperation, t.ctx, {})); + }); + + it("rejects a malformed url", async () => { + await expectValidationError(run(addOperation, t.ctx, { url: "not a url" })); + }); + + it("fails for an unknown topic", async () => { + await expect( + run(addOperation, t.ctx, { title: "A", content: "B", topic: "nope" }), + ).rejects.toBeInstanceOf(TopicNotFoundError); + }); + + it("adds a local directory from the CLI but refuses it over MCP", async () => { + const dir = mkdtempSync(join(tmpdir(), "libscope-add-op-")); + try { + writeFileSync(join(dir, "a.md"), "# A\n\nAlpha content"); + const result = await run(addOperation, t.ctx, { source: dir }); + expect(result.kind).toBe("directory"); + expect(result.documents).toHaveLength(1); + + const mcp = makeContext({ db: t.db, surface: "mcp" }); + await expectValidationError(run(addOperation, mcp.ctx, { source: dir })); + // Refused before a background task starts, also for paths that do not exist. + const api = makeContext({ db: t.db, surface: "api" }); + for (const source of [dir, join(dir, "missing")]) { + expect(() => startOperationTask(addOperation, api.ctx, { source })).toThrow( + /only available from the CLI/, + ); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + }); + + describe("get-document", () => { + it("returns paged content with tags, links and ratings", async () => { + const id = await addDoc(t, "Paged", "0123456789"); + const view = await run(getDocumentOperation, t.ctx, { + documentId: id, + offset: 2, + maxLength: 3, + }); + expect(view.content).toBe("234"); + expect(view.nextOffset).toBe(5); + expect(view.contentLength).toBe(10); + expect(view.document.documentId).toBe(id); + expect(view.ratings.totalRatings).toBe(0); + expect(view.links).toEqual({ outgoing: [], incoming: [] }); + }); + + it("rejects a negative offset", async () => { + await expectValidationError( + run(getDocumentOperation, t.ctx, { documentId: "x", offset: -1 }), + ); + }); + + it("throws NotFoundError for an unknown document", async () => { + await expect( + run(getDocumentOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("list-documents", () => { + it("pages with total and filters by library and tags", async () => { + await addDoc(t, "One", "first doc", { library: "react" }); + await addDoc(t, "Two", "second doc", { library: "react" }); + await addDoc(t, "Three", "third doc"); + const page = await run(listDocumentsOperation, t.ctx, { library: "react", limit: 1 }); + expect(page.total).toBe(2); + expect(page.items).toHaveLength(1); + expect(page.items[0]).not.toHaveProperty("content"); + expect(page.limit).toBe(1); + expect(page.offset).toBe(0); + }); + + it("rejects limit 0", async () => { + await expectValidationError(run(listDocumentsOperation, t.ctx, { limit: 0 })); + }); + }); + + describe("update-document", () => { + it("updates the title and replaces tags", async () => { + const id = await addDoc(t, "Old"); + await run(updateDocumentOperation, t.ctx, { documentId: id, tags: ["a", "b"] }); + const view = await run(updateDocumentOperation, t.ctx, { + documentId: id, + title: "New", + tags: ["b", "c"], + }); + expect(view.document.title).toBe("New"); + expect(view.tags).toEqual(["b", "c"]); + }); + + it("rejects an update with no fields", async () => { + await expectValidationError(run(updateDocumentOperation, t.ctx, { documentId: "x" })); + }); + + it("throws NotFoundError for an unknown document", async () => { + await expect( + run(updateDocumentOperation, t.ctx, { documentId: "missing", title: "x" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("delete-document", () => { + it("deletes a document", async () => { + const id = await addDoc(t, "Gone"); + expect(await run(deleteDocumentOperation, t.ctx, { documentId: id })).toEqual({ + documentId: id, + deleted: true, + }); + await expect(run(getDocumentOperation, t.ctx, { documentId: id })).rejects.toBeInstanceOf( + NotFoundError, + ); + }); + + it("rejects an empty id", async () => { + await expectValidationError(run(deleteDocumentOperation, t.ctx, { documentId: "" })); + }); + + it("throws NotFoundError for an unknown document", async () => { + await expect( + run(deleteDocumentOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("rate-document", () => { + it("records a rating attributed to the surface", async () => { + const id = await addDoc(t, "Rated"); + const mcp = makeContext({ db: t.db, surface: "mcp" }); + const rating = await run(rateDocumentOperation, mcp.ctx, { documentId: id, rating: 4 }); + expect(rating).toMatchObject({ documentId: id, rating: 4, ratedBy: "model" }); + }); + + it("rejects a rating of 6", async () => { + await expectValidationError( + run(rateDocumentOperation, t.ctx, { documentId: "x", rating: 6 }), + ); + }); + + it("throws NotFoundError for an unknown document", async () => { + await expect( + run(rateDocumentOperation, t.ctx, { documentId: "missing", rating: 3 }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("document-history and rollback-document", () => { + it("lists versions and rolls back", async () => { + const id = await addDoc(t, "Original"); + saveVersion(t.db, id); + await run(updateDocumentOperation, t.ctx, { documentId: id, title: "Changed" }); + const history = await run(documentHistoryOperation, t.ctx, { documentId: id }); + expect(history.items.length).toBeGreaterThanOrEqual(1); + const restored = await run(rollbackDocumentOperation, t.ctx, { documentId: id, version: 1 }); + expect(restored.title).toBe("Original"); + }); + + it("rejects version 0", async () => { + await expectValidationError( + run(rollbackDocumentOperation, t.ctx, { documentId: "x", version: 0 }), + ); + }); + + it("throws NotFoundError for an unknown document or version", async () => { + await expect( + run(documentHistoryOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + const id = await addDoc(t, "No versions"); + await expect( + run(rollbackDocumentOperation, t.ctx, { documentId: id, version: 9 }), + ).rejects.toBeInstanceOf(NotFoundError); + }); + }); +}); diff --git a/tests/unit/operations/graph.test.ts b/tests/unit/operations/graph.test.ts new file mode 100644 index 00000000..37006eed --- /dev/null +++ b/tests/unit/operations/graph.test.ts @@ -0,0 +1,36 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { graphOperation } from "../../../src/core/operations/index.js"; +import { createTopic } from "../../../src/core/topics.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("graph operation", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + it("returns document and topic nodes, filtered by topic name", async () => { + const topic = createTopic(t.db, { name: "Guides" }); + await addDoc(t, "In topic", undefined, { topicId: topic.id }); + await addDoc(t, "Elsewhere"); + + const all = await run(graphOperation, t.ctx); + expect(all.nodes.filter((n) => n.type === "document")).toHaveLength(2); + + const filtered = await run(graphOperation, t.ctx, { topic: "Guides" }); + const docs = filtered.nodes.filter((n) => n.type === "document"); + expect(docs.map((n) => n.label)).toEqual(["In topic"]); + }); + + it("rejects an out-of-range threshold", async () => { + await expectValidationError(run(graphOperation, t.ctx, { threshold: 2 })); + }); +}); diff --git a/tests/unit/operations/helpers.ts b/tests/unit/operations/helpers.ts new file mode 100644 index 00000000..c270102d --- /dev/null +++ b/tests/unit/operations/helpers.ts @@ -0,0 +1,83 @@ +import type Database from "better-sqlite3"; +import { expect } from "vitest"; +import type { z } from "zod"; +import type { LibScopeConfig } from "../../../src/config.js"; +import { indexDocument } from "../../../src/core/indexing.js"; +import { + createOperationContext, + runOperation, + type Operation, + type OperationContext, + type Surface, +} from "../../../src/core/operations/index.js"; +import type { LlmProvider } from "../../../src/core/rag.js"; +import { ValidationError } from "../../../src/errors.js"; +import { createTestDb } from "../../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../../fixtures/mock-provider.js"; + +/** Minimal config for operation tests (no files read). */ +export function testConfig(llmProvider?: string): LibScopeConfig { + return { + embedding: { provider: "local" }, + llm: llmProvider ? { provider: llmProvider } : {}, + database: {}, + indexing: { maxDocumentSize: 1_000_000, allowPrivateUrls: false, allowSelfSignedCerts: false }, + logging: { level: "silent" }, + } as LibScopeConfig; +} + +export interface TestContext { + db: Database.Database; + provider: MockEmbeddingProvider; + ctx: OperationContext; +} + +export function makeContext( + options: { + db?: Database.Database; + surface?: Surface; + llm?: LlmProvider | null; + config?: LibScopeConfig; + } = {}, +): TestContext { + const db = options.db ?? createTestDb(); + const provider = new MockEmbeddingProvider(); + const ctx = createOperationContext({ + db, + provider, + config: options.config ?? testConfig(), + surface: options.surface ?? "cli", + // Default: no LLM. Pass `llm: undefined` explicitly to use the configured one. + llm: "llm" in options ? options.llm : null, + }); + return { db, provider, ctx }; +} + +/** Run an operation with raw (unvalidated) input. */ +export function run( + op: Operation, + ctx: OperationContext, + input: unknown = {}, +): Promise { + return runOperation(op, ctx, input); +} + +export async function expectValidationError(promise: Promise): Promise { + await expect(promise).rejects.toBeInstanceOf(ValidationError); +} + +/** Index a small document and return its ID. */ +export async function addDoc( + t: TestContext, + title: string, + content = `${title} body text about testing operations`, + extra: { library?: string; topicId?: string } = {}, +): Promise { + const doc = await indexDocument(t.db, t.provider, { + title, + content, + sourceType: extra.library ? "library" : "manual", + ...extra, + }); + return doc.id; +} diff --git a/tests/unit/operations/links-tags-topics.test.ts b/tests/unit/operations/links-tags-topics.test.ts new file mode 100644 index 00000000..8787d47e --- /dev/null +++ b/tests/unit/operations/links-tags-topics.test.ts @@ -0,0 +1,147 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + addTagsOperation, + createTopicOperation, + deleteTopicOperation, + linkDocumentsOperation, + listLinksOperation, + listTagsOperation, + listTopicsOperation, + prerequisitesOperation, + removeTagsOperation, + suggestTagsOperation, + unlinkDocumentsOperation, +} from "../../../src/core/operations/index.js"; +import { DocumentNotFoundError, NotFoundError, TopicNotFoundError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("link, tag and topic operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + describe("links", () => { + it("links, lists with direction, follows prerequisites and unlinks", async () => { + const basics = await addDoc(t, "Basics"); + const advanced = await addDoc(t, "Advanced"); + const link = await run(linkDocumentsOperation, t.ctx, { + documentId: basics, + targetDocumentId: advanced, + linkType: "prerequisite", + }); + expect(link).toMatchObject({ sourceId: basics, targetId: advanced }); + expect(link.linkId).toEqual(expect.any(String)); + + const links = await run(listLinksOperation, t.ctx, { documentId: advanced }); + expect(links.items).toEqual([ + expect.objectContaining({ linkId: link.linkId, direction: "incoming" }), + ]); + const all = await run(listLinksOperation, t.ctx, {}); + expect(all.items).toHaveLength(1); + + const prereqs = await run(prerequisitesOperation, t.ctx, { documentId: advanced }); + expect(prereqs.items).toEqual([{ documentId: basics, title: "Basics" }]); + + await run(unlinkDocumentsOperation, t.ctx, { linkId: link.linkId }); + expect((await run(listLinksOperation, t.ctx, {})).items).toHaveLength(0); + }); + + it("rejects an unknown link type", async () => { + await expectValidationError( + run(linkDocumentsOperation, t.ctx, { + documentId: "a", + targetDocumentId: "b", + linkType: "friend", + }), + ); + }); + + it("throws NotFoundError for unknown documents and links", async () => { + const a = await addDoc(t, "A"); + await expect( + run(linkDocumentsOperation, t.ctx, { + documentId: a, + targetDocumentId: "missing", + linkType: "related", + }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + await expect(run(unlinkDocumentsOperation, t.ctx, { linkId: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + await expect( + run(listLinksOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + await expect( + run(prerequisitesOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("tags", () => { + it("adds, lists, suggests and removes tags", async () => { + const id = await addDoc(t, "Tagged", "database indexing database performance tuning"); + const added = await run(addTagsOperation, t.ctx, { documentId: id, tags: ["DB", "perf"] }); + expect(added.tags).toEqual(["db", "perf"]); + const listed = await run(listTagsOperation, t.ctx, {}); + expect(listed.items.map((tag) => tag.name)).toEqual(["db", "perf"]); + const removed = await run(removeTagsOperation, t.ctx, { documentId: id, tags: ["perf"] }); + expect(removed).toMatchObject({ removed: ["perf"], tags: ["db"] }); + const suggested = await run(suggestTagsOperation, t.ctx, { documentId: id, limit: 3 }); + expect(suggested.suggestions.length).toBeLessThanOrEqual(3); + }); + + it("rejects an empty tag list", async () => { + await expectValidationError(run(addTagsOperation, t.ctx, { documentId: "x", tags: [] })); + await expectValidationError(run(suggestTagsOperation, t.ctx, { documentId: "x", limit: 0 })); + }); + + it("throws NotFoundError for an unknown document", async () => { + for (const op of [addTagsOperation, removeTagsOperation]) { + await expect(run(op, t.ctx, { documentId: "missing", tags: ["x"] })).rejects.toBeInstanceOf( + DocumentNotFoundError, + ); + } + await expect( + run(suggestTagsOperation, t.ctx, { documentId: "missing" }), + ).rejects.toBeInstanceOf(DocumentNotFoundError); + }); + }); + + describe("topics", () => { + it("creates, lists by parent name and deletes topics", async () => { + const parent = await run(createTopicOperation, t.ctx, { name: "Backend" }); + await run(createTopicOperation, t.ctx, { name: "Databases", parent: "backend" }); + const children = await run(listTopicsOperation, t.ctx, { parent: "Backend" }); + expect(children.items.map((topic) => topic.name)).toEqual(["Databases"]); + expect(children.items[0]?.documentCount).toBe(0); + + const deleted = await run(deleteTopicOperation, t.ctx, { topic: "Databases" }); + expect(deleted.deleted).toBe(true); + expect((await run(listTopicsOperation, t.ctx, {})).items.map((x) => x.id)).toEqual([ + parent.id, + ]); + }); + + it("rejects an empty topic name", async () => { + await expectValidationError(run(createTopicOperation, t.ctx, { name: "" })); + }); + + it("throws TopicNotFoundError for unknown topics", async () => { + await expect(run(deleteTopicOperation, t.ctx, { topic: "nope" })).rejects.toBeInstanceOf( + TopicNotFoundError, + ); + await expect(run(listTopicsOperation, t.ctx, { parent: "nope" })).rejects.toBeInstanceOf( + TopicNotFoundError, + ); + }); + }); +}); diff --git a/tests/unit/operations/packs-admin.test.ts b/tests/unit/operations/packs-admin.test.ts new file mode 100644 index 00000000..044cb493 --- /dev/null +++ b/tests/unit/operations/packs-admin.test.ts @@ -0,0 +1,201 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + backupOperation, + bulkDeleteOperation, + bulkMoveOperation, + bulkRetagOperation, + createPackOperation, + dedupeOperation, + installPackOperation, + listDocumentsOperation, + listPacksOperation, + overviewOperation, + pruneExpiredOperation, + reindexOperation, + removePackOperation, + restoreOperation, +} from "../../../src/core/operations/index.js"; +import { indexDocument } from "../../../src/core/indexing.js"; +import { createTopic } from "../../../src/core/topics.js"; +import { NotFoundError, TopicNotFoundError, ValidationError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("pack and admin operations", () => { + let t: TestContext; + let dir: string; + + beforeEach(() => { + t = makeContext(); + dir = mkdtempSync(join(tmpdir(), "libscope-ops-")); + }); + + afterEach(() => { + t.db.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + function writePack(name: string): string { + const path = join(dir, `${name}.json`); + writeFileSync( + path, + JSON.stringify({ + name, + version: "1.0.0", + description: "Test pack", + documents: [{ title: "Pack doc", content: "Content from a pack", source: "test" }], + metadata: { author: "tests", license: "MIT", createdAt: new Date().toISOString() }, + }), + ); + return path; + } + + describe("packs", () => { + it("installs from a file, lists, and removes", async () => { + const result = await run(installPackOperation, t.ctx, { pack: writePack("demo") }); + expect(result).toMatchObject({ packName: "demo", documentsInstalled: 1 }); + const listed = await run(listPacksOperation, t.ctx, {}); + expect(listed.items.map((p) => p.name)).toEqual(["demo"]); + await run(removePackOperation, t.ctx, { pack: "demo" }); + expect((await run(listPacksOperation, t.ctx, {})).items).toEqual([]); + }); + + it("refuses pack files over the REST API", async () => { + const api = makeContext({ db: t.db, surface: "api" }); + await expectValidationError(run(installPackOperation, api.ctx, { pack: writePack("x") })); + await expectValidationError(run(createPackOperation, api.ctx, { name: "x" })); + }); + + it("rejects an empty pack name", async () => { + await expectValidationError(run(installPackOperation, t.ctx, { pack: "" })); + await expectValidationError(run(removePackOperation, t.ctx, {})); + }); + + it("throws NotFoundError when removing a pack that is not installed", async () => { + await expect(run(removePackOperation, t.ctx, { pack: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + }); + + it("creates a pack file from the database", async () => { + await addDoc(t, "Packed"); + const outputPath = join(dir, "out.json"); + const pack = await run(createPackOperation, t.ctx, { name: "out", outputPath }); + expect(pack.documents).toHaveLength(1); + expect(existsSync(outputPath)).toBe(true); + }); + }); + + describe("overview", () => { + it("reports counts, topics, packs and health", async () => { + createTopic(t.db, { name: "Guides" }); + await addDoc(t, "Doc"); + const overview = await run(overviewOperation, t.ctx, {}); + expect(overview.stats.totalDocuments).toBe(1); + expect(overview.topics.map((x) => x.name)).toEqual(["Guides"]); + expect(overview.health).toMatchObject({ database: "ok", fts: "ok" }); + expect(overview.index.configured).toMatchObject({ provider: "mock", dimensions: 4 }); + }); + }); + + describe("reindex and dedupe", () => { + it("re-embeds every chunk", async () => { + await addDoc(t, "Reindexed"); + const result = await run(reindexOperation, t.ctx, {}); + expect(result.total).toBeGreaterThan(0); + expect(result.failed).toBe(0); + }); + + it("rejects rebuild combined with filters, and bad dates", async () => { + await expectValidationError( + run(reindexOperation, t.ctx, { rebuild: true, documentIds: ["a"] }), + ); + await expectValidationError(run(reindexOperation, t.ctx, { since: "yesterday" })); + }); + + it("groups exact duplicates", async () => { + await indexDocument(t.db, t.provider, { + title: "Copy one", + content: "identical body", + sourceType: "manual", + }); + await indexDocument(t.db, t.provider, { + title: "Copy two", + content: "identical body", + sourceType: "manual", + dedup: "force", + }); + const result = await run(dedupeOperation, t.ctx, { strategy: "exact" }); + expect(result.items).toHaveLength(1); + await expectValidationError(run(dedupeOperation, t.ctx, { threshold: 2 })); + }); + }); + + describe("backup and restore", () => { + it("writes and reads a backup from the CLI, but not over MCP", async () => { + await addDoc(t, "Saved"); + const outputPath = join(dir, "backup.json"); + const backup = await run(backupOperation, t.ctx, { outputPath }); + expect(backup.counts.documents).toBe(1); + + const target = makeContext(); + try { + const restored = await run(restoreOperation, target.ctx, { backupPath: outputPath }); + expect(restored.counts.documents).toBe(1); + } finally { + target.db.close(); + } + + const mcp = makeContext({ db: t.db, surface: "mcp" }); + await expectValidationError(run(backupOperation, mcp.ctx, { outputPath })); + await expectValidationError(run(restoreOperation, t.ctx, {})); + }); + }); + + describe("prune-expired", () => { + it("deletes expired documents only", async () => { + await indexDocument(t.db, t.provider, { + title: "Old news", + content: "expired content", + sourceType: "manual", + expiresAt: "2000-01-01T00:00:00Z", + }); + await addDoc(t, "Fresh"); + expect(await run(pruneExpiredOperation, t.ctx, {})).toEqual({ pruned: 1 }); + expect((await run(listDocumentsOperation, t.ctx, {})).total).toBe(1); + }); + }); + + describe("bulk operations", () => { + it("deletes (with dry run), retags and moves by filter", async () => { + createTopic(t.db, { name: "Target" }); + await addDoc(t, "R1", "react one", { library: "react" }); + await addDoc(t, "R2", "react two", { library: "react" }); + + const dry = await run(bulkDeleteOperation, t.ctx, { library: "react", dryRun: true }); + expect(dry.affected).toBe(2); + const retag = await run(bulkRetagOperation, t.ctx, { library: "react", addTags: ["ui"] }); + expect(retag.affected).toBe(2); + const moved = await run(bulkMoveOperation, t.ctx, { tags: ["ui"], targetTopic: "Target" }); + expect(moved.affected).toBe(2); + const deleted = await run(bulkDeleteOperation, t.ctx, { topic: "target" }); + expect(deleted.affected).toBe(2); + }); + + it("rejects an empty selector and an invalid source type", async () => { + await expect(run(bulkDeleteOperation, t.ctx, {})).rejects.toBeInstanceOf(ValidationError); + await expectValidationError(run(bulkDeleteOperation, t.ctx, { sourceType: "web" })); + }); + + it("throws TopicNotFoundError for an unknown target topic", async () => { + await expect( + run(bulkMoveOperation, t.ctx, { library: "x", targetTopic: "nowhere" }), + ).rejects.toBeInstanceOf(TopicNotFoundError); + }); + }); +}); diff --git a/tests/unit/operations/registries.test.ts b/tests/unit/operations/registries.test.ts new file mode 100644 index 00000000..90041a0a --- /dev/null +++ b/tests/unit/operations/registries.test.ts @@ -0,0 +1,154 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { gzipSync } from "node:zlib"; +import { execFileSync } from "node:child_process"; +import { NotFoundError, ValidationError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; + +// Registries live under ~/.libscope/registries: point homedir() at a temp directory. +let tempHome = tmpdir(); +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const ops = await import("../../../src/core/operations/index.js"); +const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); +const { makeContext, run, expectValidationError } = await import("./helpers.js"); +type TestContext = ReturnType; + +initLogger("silent"); + +describe("registry operations", () => { + let t: TestContext; + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "libscope-registry-ops-")); + tempHome = join(dir, "home"); + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + function writeGzPack(name: string): string { + const path = join(dir, `${name}.json.gz`); + const pack = { + name, + version: "1.0.0", + description: `The ${name} pack`, + documents: [{ title: "Pack doc", content: "Content from a registry pack", source: "t" }], + metadata: { author: "tests", license: "MIT", createdAt: "2026-01-01T00:00:00.000Z" }, + }; + writeFileSync(path, gzipSync(JSON.stringify(pack))); + return path; + } + + /** create-registry, then a bare clone to push to; returns its file:/// URL. */ + async function makeRemote(): Promise { + const created = await run(ops.createRegistryOperation, t.ctx, { path: join(dir, "src") }); + const bare = join(dir, "packs.git"); + execFileSync("git", ["clone", "--quiet", "--bare", created.path, bare]); + return `file://${bare}`; + } + + it("adds, publishes, lists, searches, installs, unpublishes and removes", async () => { + const url = await makeRemote(); + const added = await run(ops.addRegistryOperation, t.ctx, { url }); + expect(added).toMatchObject({ name: "packs", sync: { status: "success", packs: 0 } }); + + const published = await run(ops.publishPackOperation, t.ctx, { + file: writeGzPack("demo"), + registry: "packs", + }); + expect(published).toMatchObject({ packName: "demo", version: "1.0.0", branch: null }); + + const registries = await run(ops.listRegistriesOperation, t.ctx, {}); + expect(registries.items).toEqual([ + expect.objectContaining({ name: "packs", url, packs: 1 }) as unknown, + ]); + + const found = await run(ops.searchRegistriesOperation, t.ctx, { query: "demo" }); + expect(found.items.map((p) => [p.registry, p.name])).toEqual([["packs", "demo"]]); + + const available = await run(ops.listPacksOperation, t.ctx, { available: true }); + expect(available).toMatchObject({ items: [{ name: "demo", registry: "packs" }] }); + + const installed = await run(ops.installPackOperation, t.ctx, { pack: "demo@1.0.0" }); + expect(installed).toMatchObject({ packName: "demo", documentsInstalled: 1 }); + + await expectValidationError( + run(ops.unpublishPackOperation, t.ctx, { pack: "demo", registry: "packs" }), + ); + expect( + await run(ops.unpublishPackOperation, t.ctx, { pack: "demo@1.0.0", registry: "packs" }), + ).toMatchObject({ removed: true }); + expect((await run(ops.searchRegistriesOperation, t.ctx, { query: "demo" })).items).toEqual([]); + + await run(ops.removeRegistryOperation, t.ctx, { name: "packs" }); + expect(existsSync(getRegistryCacheDir("packs"))).toBe(false); + expect((await run(ops.listRegistriesOperation, t.ctx, {})).items).toEqual([]); + }); + + it("syncs one or all registries and reports registries that cannot be reached", async () => { + const url = await makeRemote(); + await run(ops.addRegistryOperation, t.ctx, { url, name: "first", sync: false }); + await run(ops.addRegistryOperation, t.ctx, { + url: `file://${join(dir, "missing.git")}`, + name: "broken", + sync: false, + }); + expect((await run(ops.listPacksOperation, t.ctx, { available: true })).warnings).toHaveLength( + 2, + ); + + const one = await run(ops.syncRegistriesOperation, t.ctx, { name: "first" }); + expect(one.items).toMatchObject([{ registry: "first", status: "success", packs: 0 }]); + const all = await run(ops.syncRegistriesOperation, t.ctx, {}); + expect(all.items.map((s) => s.status)).toEqual(["success", "error"]); + }); + + it("throws NotFoundError for unknown registries and packs", async () => { + await expect(run(ops.installPackOperation, t.ctx, { pack: "demo" })).rejects.toBeInstanceOf( + NotFoundError, + ); + for (const [op, input] of [ + [ops.syncRegistriesOperation, { name: "nope" }], + [ops.searchRegistriesOperation, { query: "x", registry: "nope" }], + [ops.removeRegistryOperation, { name: "nope" }], + [ops.listPacksOperation, { available: true, registry: "nope" }], + [ops.installPackOperation, { pack: "demo", registry: "nope" }], + ] as const) { + await expect(run(op, t.ctx, input), op.name).rejects.toBeInstanceOf(NotFoundError); + } + }); + + it("validates input", async () => { + await expectValidationError(run(ops.listPacksOperation, t.ctx, { registry: "packs" })); + await expectValidationError( + run(ops.installPackOperation, t.ctx, { pack: "a.json", registry: "packs" }), + ); + await expectValidationError(run(ops.addRegistryOperation, t.ctx, { url: "/srv/x.git" })); + await expectValidationError(run(ops.searchRegistriesOperation, t.ctx, { query: "" })); + }); + + it("reads and writes local files only from the CLI and the Node.js API", async () => { + for (const surface of ["api", "mcp"] as const) { + const other = makeContext({ db: t.db, surface }); + await expect( + run(ops.installPackOperation, other.ctx, { pack: writeGzPack("x") }), + ).rejects.toBeInstanceOf(ValidationError); + await expect( + run(ops.publishPackOperation, other.ctx, { file: "x.json", registry: "packs" }), + ).rejects.toThrow(/only available from the CLI/); + await expect( + run(ops.createRegistryOperation, other.ctx, { path: join(dir, "r") }), + ).rejects.toThrow(/only available from the CLI/); + } + }); +}); diff --git a/tests/unit/operations/registry.test.ts b/tests/unit/operations/registry.test.ts new file mode 100644 index 00000000..4134eb3d --- /dev/null +++ b/tests/unit/operations/registry.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from "vitest"; +import { z } from "zod"; +import { OPERATIONS, getOperation, runOperation } from "../../../src/core/operations/index.js"; +import { ValidationError } from "../../../src/errors.js"; +import { makeContext } from "./helpers.js"; + +describe("OPERATIONS", () => { + it("has unique kebab-case names", () => { + const names = OPERATIONS.map((op) => op.name); + expect(new Set(names).size).toBe(names.length); + for (const name of names) expect(name).toMatch(/^[a-z]+(?:-[a-z]+)*$/); + }); + + it("describes every input field", () => { + const missing = OPERATIONS.flatMap((op) => + Object.entries(op.input.shape) + .filter(([, schema]) => !(schema as z.ZodType).description) + .map(([field]) => `${op.name}.${field}`), + ); + expect(missing).toEqual([]); + }); + + it("maps every :param in an HTTP path to an input field, with unique routes", () => { + const routes = new Set(); + for (const op of OPERATIONS) { + if (!op.http) continue; + const route = `${op.http.method} ${op.http.path}`; + expect(routes.has(route), route).toBe(false); + routes.add(route); + for (const segment of op.http.path.split("/")) { + if (segment.startsWith(":")) expect(op.input.shape, route).toHaveProperty(segment.slice(1)); + } + } + }); + + it("produces JSON Schema for every input (for MCP and OpenAPI)", () => { + for (const op of OPERATIONS) { + expect(() => z.toJSONSchema(op.input, { io: "input" }), op.name).not.toThrow(); + } + }); + + it("looks operations up by name", () => { + expect(getOperation("search")?.group).toBe("search"); + expect(getOperation("nope")).toBeUndefined(); + }); + + it("turns non-object input into a ValidationError naming the operation", async () => { + const { db, ctx } = makeContext(); + try { + const op = getOperation("get-document")!; + await expect(runOperation(op, ctx, "doc-1")).rejects.toBeInstanceOf(ValidationError); + await expect(runOperation(op, ctx, { documentId: 5 })).rejects.toThrow( + "Invalid input for get-document: documentId:", + ); + } finally { + db.close(); + } + }); +}); diff --git a/tests/unit/operations/search.test.ts b/tests/unit/operations/search.test.ts new file mode 100644 index 00000000..70407b3f --- /dev/null +++ b/tests/unit/operations/search.test.ts @@ -0,0 +1,118 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { createDatabase } from "../../../src/db/connection.js"; +import { createVectorTable, runMigrations } from "../../../src/db/schema.js"; +import { askOperation, searchOperation } from "../../../src/core/operations/index.js"; +import type { LlmProvider } from "../../../src/core/rag.js"; +import { ConfigError, DocumentNotFoundError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { MockEmbeddingProvider } from "../../fixtures/mock-provider.js"; +import { + addDoc, + expectValidationError, + makeContext, + run, + testConfig, + type TestContext, +} from "./helpers.js"; + +initLogger("silent"); + +const fakeLlm: LlmProvider = { + model: "fake-model", + complete: (prompt: string) => + Promise.resolve({ text: `answered (${prompt.length} chars)`, tokensUsed: 7 }), +}; + +describe("search operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + describe("search", () => { + it("finds documents by keyword and returns documentId and chunkId", async () => { + const id = await addDoc(t, "Kubernetes", "Kubernetes pods and deployments explained"); + await addDoc(t, "Cooking", "Recipes for pasta"); + const result = await run(searchOperation, t.ctx, { query: "kubernetes" }); + expect(result.total).toBeGreaterThanOrEqual(1); + expect(result.items[0]).toMatchObject({ documentId: id }); + expect(result.items[0]?.chunkId).toEqual(expect.any(String)); + expect(result.limit).toBe(10); + }); + + it("needs exactly one of query and relatedTo", async () => { + await expectValidationError(run(searchOperation, t.ctx, {})); + await expectValidationError(run(searchOperation, t.ctx, { query: "a", relatedTo: "b" })); + }); + + it("rejects limit above the maximum", async () => { + await expectValidationError(run(searchOperation, t.ctx, { query: "a", limit: 500 })); + }); + + it("throws NotFoundError when relatedTo names nothing", async () => { + await expect(run(searchOperation, t.ctx, { relatedTo: "missing" })).rejects.toBeInstanceOf( + DocumentNotFoundError, + ); + }); + + it("finds content related to a document (sqlite-vec)", async () => { + const db = createDatabase(":memory:"); + runMigrations(db); + createVectorTable(db, new MockEmbeddingProvider()); + const v = makeContext({ db }); + try { + const id = await addDoc(v, "Seed", "Vector search seed document"); + await addDoc(v, "Other", "Another document to compare with"); + const result = await run(searchOperation, v.ctx, { relatedTo: id }); + expect(result.items.every((r) => r.documentId !== id)).toBe(true); + expect(result.total).toBe(result.items.length); + } finally { + db.close(); + } + }); + }); + + describe("ask", () => { + it("answers with the LLM and lists sources", async () => { + const llmCtx = makeContext({ db: t.db, llm: fakeLlm }); + await addDoc(t, "Install", "Run npm install to install libscope"); + const result = await run(askOperation, llmCtx.ctx, { question: "How do I install?" }); + expect(result.mode).toBe("answer"); + if (result.mode === "answer") { + expect(result.model).toBe("fake-model"); + expect(result.sources.length).toBeGreaterThan(0); + } + }); + + it("returns context in passthrough mode", async () => { + const p = makeContext({ db: t.db, config: testConfig("passthrough"), llm: undefined }); + await addDoc(t, "Install", "Run npm install to install libscope"); + const result = await run(askOperation, p.ctx, { question: "install" }); + expect(result.mode).toBe("context"); + if (result.mode === "context") expect(result.contextPrompt).toContain("install"); + }); + + it("resolves llm.provider auto to passthrough under MCP", async () => { + const mcp = makeContext({ db: t.db, surface: "mcp", llm: undefined }); + await addDoc(t, "Install", "Run npm install to install libscope"); + const result = await run(askOperation, mcp.ctx, { question: "install" }); + expect(result.mode).toBe("context"); + }); + + it("throws ConfigError when no LLM is configured", async () => { + await expect(run(askOperation, t.ctx, { question: "anything" })).rejects.toBeInstanceOf( + ConfigError, + ); + }); + + it("rejects an empty question and topK above 20", async () => { + await expectValidationError(run(askOperation, t.ctx, { question: "" })); + await expectValidationError(run(askOperation, t.ctx, { question: "q", topK: 21 })); + }); + }); +}); diff --git a/tests/unit/operations/searches-analytics.test.ts b/tests/unit/operations/searches-analytics.test.ts new file mode 100644 index 00000000..943349fa --- /dev/null +++ b/tests/unit/operations/searches-analytics.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + deleteSavedSearchOperation, + listSavedSearchesOperation, + popularOperation, + runSavedSearchOperation, + saveSearchOperation, + searchAnalyticsOperation, + searchOperation, + staleOperation, + topQueriesOperation, +} from "../../../src/core/operations/index.js"; +import { NotFoundError, ValidationError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +initLogger("silent"); + +describe("saved search and analytics operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + t.db.close(); + }); + + describe("saved searches", () => { + it("saves with filters, lists, runs and deletes", async () => { + await addDoc(t, "Hooks", "React hooks guide", { library: "react" }); + const saved = await run(saveSearchOperation, t.ctx, { + name: "react-hooks", + query: "hooks", + library: "react", + sourceType: "library", + }); + expect(saved.filters).toEqual({ library: "react", sourceType: "library" }); + + const list = await run(listSavedSearchesOperation, t.ctx, {}); + expect(list).toMatchObject({ total: 1, limit: 50, offset: 0 }); + + const ran = await run(runSavedSearchOperation, t.ctx, { search: "react-hooks" }); + expect(ran.items).toHaveLength(1); + expect(ran.search.resultCount).toBe(1); + expect(ran.search.lastRunAt).not.toBeNull(); + + // Stored filters apply when the search runs. + await run(saveSearchOperation, t.ctx, { name: "vue", query: "hooks", library: "vue" }); + expect((await run(runSavedSearchOperation, t.ctx, { search: "vue" })).items).toEqual([]); + + await run(deleteSavedSearchOperation, t.ctx, { search: saved.id }); + expect((await run(listSavedSearchesOperation, t.ctx, {})).total).toBe(1); + }); + + it("rejects a missing query and a duplicate name", async () => { + await expectValidationError(run(saveSearchOperation, t.ctx, { name: "x" })); + await run(saveSearchOperation, t.ctx, { name: "dup", query: "q" }); + await expect( + run(saveSearchOperation, t.ctx, { name: "dup", query: "q" }), + ).rejects.toBeInstanceOf(ValidationError); + }); + + it("throws NotFoundError for an unknown saved search", async () => { + await expect(run(runSavedSearchOperation, t.ctx, { search: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + await expect( + run(deleteSavedSearchOperation, t.ctx, { search: "nope" }), + ).rejects.toBeInstanceOf(NotFoundError); + }); + }); + + describe("analytics", () => { + it("reports popular documents, stale documents, top queries and search analytics", async () => { + const id = await addDoc(t, "Popular", "frequently searched content"); + await addDoc(t, "Lonely", "nobody searches this"); + await run(searchOperation, t.ctx, { query: "frequently" }); + await run(searchOperation, t.ctx, { query: "frequently" }); + + const popular = await run(popularOperation, t.ctx, {}); + expect(popular.items[0]).toMatchObject({ documentId: id }); + const stale = await run(staleOperation, t.ctx, { days: 30 }); + expect(stale.items.map((d) => d.title)).toEqual(["Lonely"]); + const top = await run(topQueriesOperation, t.ctx, { limit: 5 }); + expect(top.items[0]).toMatchObject({ query: "frequently", count: 2 }); + const analytics = await run(searchAnalyticsOperation, t.ctx, {}); + expect(analytics.totalSearches).toBe(2); + expect(analytics.knowledgeGaps).toEqual(expect.any(Array)); + }); + + it("rejects out-of-range input", async () => { + await expectValidationError(run(popularOperation, t.ctx, { limit: 0 })); + await expectValidationError(run(staleOperation, t.ctx, { days: 0 })); + await expectValidationError(run(topQueriesOperation, t.ctx, { limit: "ten" })); + await expectValidationError(run(searchAnalyticsOperation, t.ctx, { days: 1.5 })); + }); + }); +}); diff --git a/tests/unit/operations/webhooks-tasks.test.ts b/tests/unit/operations/webhooks-tasks.test.ts new file mode 100644 index 00000000..12ebcaa8 --- /dev/null +++ b/tests/unit/operations/webhooks-tasks.test.ts @@ -0,0 +1,134 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { z } from "zod"; +import { NotFoundError } from "../../../src/errors.js"; +import { initLogger } from "../../../src/logger.js"; +import { addDoc, expectValidationError, makeContext, run, type TestContext } from "./helpers.js"; + +vi.mock("node:dns", async (importOriginal) => { + const actual: typeof import("node:dns") = await importOriginal(); + return { + ...actual, + promises: { + ...actual.promises, + resolve4: vi.fn().mockResolvedValue(["93.184.216.34"]), + resolve6: vi.fn().mockResolvedValue([]), + }, + }; +}); + +const ops = await import("../../../src/core/operations/index.js"); + +initLogger("silent"); + +describe("webhook and task operations", () => { + let t: TestContext; + + beforeEach(() => { + t = makeContext(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + t.db.close(); + }); + + describe("webhooks", () => { + it("creates, lists, tests and deletes a webhook", async () => { + const created = await run(ops.createWebhookOperation, t.ctx, { + url: "https://hooks.example.com/libscope", + events: ["document.created"], + }); + expect(created).not.toHaveProperty("secret"); + const listed = await run(ops.listWebhooksOperation, t.ctx, {}); + expect(listed.items.map((w) => w.id)).toEqual([created.id]); + + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response(null, { status: 204, statusText: "No Content" })), + ); + const tested = await run(ops.testWebhookOperation, t.ctx, { webhookId: created.id }); + expect(tested).toMatchObject({ status: 204 }); + + await run(ops.deleteWebhookOperation, t.ctx, { webhookId: created.id }); + expect((await run(ops.listWebhooksOperation, t.ctx, {})).items).toEqual([]); + }); + + it("rejects unknown events and non-http URLs", async () => { + await expectValidationError( + run(ops.createWebhookOperation, t.ctx, { + url: "https://hooks.example.com", + events: ["document.exploded"], + }), + ); + await expectValidationError( + run(ops.createWebhookOperation, t.ctx, { + url: "ftp://hooks.example.com", + events: ["document.created"], + }), + ); + }); + + it("throws NotFoundError for an unknown webhook", async () => { + await expect( + run(ops.deleteWebhookOperation, t.ctx, { webhookId: "nope" }), + ).rejects.toBeInstanceOf(NotFoundError); + await expect( + run(ops.testWebhookOperation, t.ctx, { webhookId: "nope" }), + ).rejects.toBeInstanceOf(NotFoundError); + }); + }); + + describe("tasks", () => { + it("runs an operation as a background task and reports its result", async () => { + await addDoc(t, "Counted"); + const task = ops.startOperationTask(ops.overviewOperation, t.ctx, {}); + expect(task.operation).toBe("overview"); + await vi.waitFor(() => { + expect(ops.getTaskOperation.handler(t.ctx, { taskId: task.id }).status).toBe("completed"); + }); + const done = await run(ops.getTaskOperation, t.ctx, { taskId: task.id }); + expect(JSON.parse(done.result ?? "{}")).toMatchObject({ stats: { totalDocuments: 1 } }); + const listed = await run(ops.listTasksOperation, t.ctx, {}); + expect(listed.items.some((x) => x.id === task.id)).toBe(true); + const cancel = await run(ops.cancelTaskOperation, t.ctx, { taskId: task.id }); + expect(cancel).toMatchObject({ cancelRequested: false, status: "completed" }); + }); + + it("cancels a running task", async () => { + const slow = ops.defineOperation({ + name: "slow", + group: "admin", + summary: "Waits until cancelled", + input: z.object({}), + handler: (ctx) => + new Promise((_resolve, reject) => { + ctx.signal?.addEventListener("abort", () => + reject(new DOMException("aborted", "AbortError")), + ); + }), + }); + const task = ops.startOperationTask(slow, t.ctx, {}); + const cancel = await run(ops.cancelTaskOperation, t.ctx, { taskId: task.id }); + expect(cancel.cancelRequested).toBe(true); + await vi.waitFor(() => { + expect(ops.getTaskOperation.handler(t.ctx, { taskId: task.id }).status).toBe("cancelled"); + }); + }); + + it("validates input before starting a task", () => { + expect(() => ops.startOperationTask(ops.reindexOperation, t.ctx, { batchSize: 0 })).toThrow( + "Invalid input for reindex", + ); + }); + + it("rejects an empty task id and throws NotFoundError for an unknown one", async () => { + await expectValidationError(run(ops.getTaskOperation, t.ctx, { taskId: "" })); + await expect(run(ops.getTaskOperation, t.ctx, { taskId: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + await expect(run(ops.cancelTaskOperation, t.ctx, { taskId: "nope" })).rejects.toBeInstanceOf( + NotFoundError, + ); + }); + }); +}); diff --git a/tests/unit/packs.test.ts b/tests/unit/packs.test.ts index 9ca8dd4a..7be080ae 100644 --- a/tests/unit/packs.test.ts +++ b/tests/unit/packs.test.ts @@ -11,7 +11,6 @@ import { removePack, listInstalledPacks, createPack, - listAvailablePacks, createPackFromSource, } from "../../src/core/packs.js"; import type { KnowledgePack } from "../../src/core/packs.js"; @@ -76,7 +75,7 @@ describe("knowledge packs", () => { const docs = db .prepare("SELECT id, pack_name FROM documents WHERE pack_name = ?") .all("test-pack") as Array<{ id: string; pack_name: string }>; - expect(docs.length).toBe(2); + expect(docs).toHaveLength(2); // Verify pack is recorded const packs = db.prepare("SELECT * FROM packs WHERE name = ?").all("test-pack") as Array<{ @@ -84,7 +83,7 @@ describe("knowledge packs", () => { version: string; doc_count: number; }>; - expect(packs.length).toBe(1); + expect(packs).toHaveLength(1); expect(packs[0]!.version).toBe("1.0.0"); expect(packs[0]!.doc_count).toBe(2); }); @@ -125,6 +124,94 @@ describe("knowledge packs", () => { }); }); + describe("installPack — cancellation", () => { + function writeManyDocPack(count: number): string { + const documents = Array.from({ length: count }, (_, i) => ({ + title: `Doc ${i}`, + content: `# Doc ${i}\n\nBody of document ${i}.`, + source: "", + })); + const packPath = join(tempDir, "many.json"); + writeFileSync(packPath, JSON.stringify(makeSamplePack({ documents })), "utf-8"); + return packPath; + } + + function countRows(table: "documents" | "packs" | "chunks"): number { + return (db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n; + } + + it.each([1, 4])( + "rejects with the abort reason and rolls back when aborted mid-install (concurrency=%i)", + async (concurrency) => { + const packPath = writeManyDocPack(6); + const controller = new AbortController(); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + try { + const promise = installPack(db, provider, packPath, { + batchSize: 1, + concurrency, + signal: controller.signal, + onProgress: () => controller.abort(), + }); + await expect(promise).rejects.toHaveProperty("name", "AbortError"); + await new Promise((r) => setTimeout(r, 20)); + } finally { + process.off("unhandledRejection", onUnhandled); + } + expect(unhandled).toEqual([]); + expect(countRows("packs")).toBe(0); + expect(countRows("documents")).toBe(0); + expect(countRows("chunks")).toBe(0); + }, + ); + + it("rejects instead of hanging when onProgress throws", async () => { + const packPath = writeManyDocPack(4); + const promise = installPack(db, provider, packPath, { + batchSize: 1, + onProgress: () => { + throw new Error("progress sink failed"); + }, + }); + await expect(promise).rejects.toThrow("progress sink failed"); + expect(countRows("packs")).toBe(0); + expect(countRows("documents")).toBe(0); + }); + + it("does not create the pack when the signal is already aborted", async () => { + const packPath = writeManyDocPack(2); + const controller = new AbortController(); + controller.abort(); + const embedSpy = vi.spyOn(provider, "embedBatch"); + await expect(installPack(db, provider, packPath, { signal: controller.signal })).rejects.toBe( + controller.signal.reason, + ); + expect(embedSpy).not.toHaveBeenCalled(); + expect(countRows("packs")).toBe(0); + }); + + it("can be installed again after a cancelled install", async () => { + const packPath = writeManyDocPack(3); + const controller = new AbortController(); + await expect( + installPack(db, provider, packPath, { + batchSize: 1, + concurrency: 1, + signal: controller.signal, + onProgress: () => controller.abort(), + }), + ).rejects.toHaveProperty("name", "AbortError"); + + const result = await installPack(db, provider, packPath); + expect(result.alreadyInstalled).toBe(false); + expect(result.documentsInstalled).toBe(3); + }); + }); + describe("removePack", () => { it("should remove a pack and its associated documents", async () => { const pack = makeSamplePack(); @@ -137,17 +224,17 @@ describe("knowledge packs", () => { const docsBefore = db .prepare("SELECT id FROM documents WHERE pack_name = ?") .all("test-pack"); - expect(docsBefore.length).toBe(2); + expect(docsBefore).toHaveLength(2); removePack(db, "test-pack"); // Verify docs removed const docsAfter = db.prepare("SELECT id FROM documents WHERE pack_name = ?").all("test-pack"); - expect(docsAfter.length).toBe(0); + expect(docsAfter).toHaveLength(0); // Verify pack record removed const packRecord = db.prepare("SELECT * FROM packs WHERE name = ?").all("test-pack"); - expect(packRecord.length).toBe(0); + expect(packRecord).toHaveLength(0); }); it("should throw when removing a non-existent pack", () => { @@ -173,7 +260,7 @@ describe("knowledge packs", () => { await installPack(db, provider, path2); const packs = listInstalledPacks(db); - expect(packs.length).toBe(2); + expect(packs).toHaveLength(2); expect(packs.map((p) => p.name)).toEqual(["pack-a", "pack-b"]); expect(packs[0]!.version).toBe("1.0.0"); expect(packs[0]!.docCount).toBe(2); @@ -201,7 +288,7 @@ describe("knowledge packs", () => { }); expect(pack.name).toBe("exported"); - expect(pack.documents.length).toBe(2); + expect(pack.documents).toHaveLength(2); expect(pack.version).toBe("1.0.0"); expect(pack.metadata.author).toBe("libscope"); @@ -408,42 +495,6 @@ describe("knowledge packs", () => { /must be within the current working directory/, ); }); - - it("should reject http registry URLs", async () => { - await expect( - installPack(db, provider, "some-pack", { - registryUrl: "http://evil.com/registry.json", - }), - ).rejects.toThrow(/must use https/); - }); - - it("should reject private IP registry URLs", async () => { - await expect( - installPack(db, provider, "some-pack", { - registryUrl: "https://127.0.0.1/registry.json", - }), - ).rejects.toThrow(/private/); - }); - - it("should reject localhost registry URLs", async () => { - await expect( - installPack(db, provider, "some-pack", { - registryUrl: "https://localhost/registry.json", - }), - ).rejects.toThrow(/private/); - }); - - it("should reject http registry URL in listAvailablePacks", async () => { - await expect(listAvailablePacks("http://evil.com/registry.json")).rejects.toThrow( - /must use https/, - ); - }); - - it("should reject private IP in listAvailablePacks", async () => { - await expect(listAvailablePacks("https://192.168.1.1/registry.json")).rejects.toThrow( - /private/, - ); - }); }); describe("createPackFromSource", () => { @@ -955,7 +1006,7 @@ describe("knowledge packs", () => { const docs = db .prepare("SELECT id FROM documents WHERE pack_name = ?") .all("concurrent-4") as Array<{ id: string }>; - expect(docs.length).toBe(4); + expect(docs).toHaveLength(4); }); it("should make multiple embedBatch calls with small batchSize and high concurrency", async () => { diff --git a/tests/unit/plugin-provider.test.ts b/tests/unit/plugin-provider.test.ts deleted file mode 100644 index 580a286f..00000000 --- a/tests/unit/plugin-provider.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { describe, it, expect, beforeEach } from "vitest"; -import type { EmbeddingProvider } from "../../src/providers/embedding.js"; -import type { ProviderFactory } from "../../src/providers/index.js"; -import { registerProvider, createEmbeddingProvider } from "../../src/providers/index.js"; -import type { LibScopeConfig } from "../../src/config.js"; - -function makeConfig(provider: string): LibScopeConfig { - return { - embedding: { - provider, - }, - database: { path: ":memory:" }, - indexing: { maxDocumentSize: 1024 }, - logging: { level: "silent" }, - }; -} - -class StubProvider implements EmbeddingProvider { - readonly name = "stub"; - readonly dimensions = 3; - - embed(_text: string): Promise { - return Promise.resolve([0.1, 0.2, 0.3]); - } - - embedBatch(texts: string[]): Promise { - return Promise.resolve(texts.map(() => [0.1, 0.2, 0.3])); - } -} - -describe("plugin provider registration", () => { - beforeEach(() => { - // Register a fresh custom provider before each test - registerProvider("stub", () => new StubProvider()); - }); - - it("should use a registered custom provider", () => { - const provider = createEmbeddingProvider(makeConfig("stub")); - expect(provider.name).toBe("stub"); - expect(provider.dimensions).toBe(3); - }); - - it("should return embeddings from a custom provider", async () => { - const provider = createEmbeddingProvider(makeConfig("stub")); - const result = await provider.embed("hello"); - expect(result).toEqual([0.1, 0.2, 0.3]); - }); - - it("should return batch embeddings from a custom provider", async () => { - const provider = createEmbeddingProvider(makeConfig("stub")); - const results = await provider.embedBatch(["a", "b"]); - expect(results).toHaveLength(2); - }); - - it("should pass config to the factory function", () => { - let receivedConfig: LibScopeConfig | undefined; - const factory: ProviderFactory = (config) => { - receivedConfig = config; - return new StubProvider(); - }; - registerProvider("config-test", factory); - const config = makeConfig("config-test"); - createEmbeddingProvider(config); - expect(receivedConfig).toBe(config); - }); - - it("should still support built-in providers", () => { - const provider = createEmbeddingProvider(makeConfig("local")); - expect(provider.name).toBe("local"); - }); - - it("should throw for unknown unregistered providers", () => { - expect(() => createEmbeddingProvider(makeConfig("nonexistent"))).toThrow( - "Unknown embedding provider", - ); - }); -}); diff --git a/tests/unit/rag-answer.test.ts b/tests/unit/rag-answer.test.ts new file mode 100644 index 00000000..437e677f --- /dev/null +++ b/tests/unit/rag-answer.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import type Database from "better-sqlite3"; +import { createTestDb } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { answer, type LlmProvider } from "../../src/core/rag.js"; +import { indexDocument } from "../../src/core/indexing.js"; +import { addTagsToDocument } from "../../src/core/tags.js"; +import { ConfigError } from "../../src/errors.js"; +import { initLogger } from "../../src/logger.js"; + +initLogger("silent"); + +describe("answer", () => { + let db: Database.Database; + let provider: MockEmbeddingProvider; + + beforeEach(async () => { + db = createTestDb(); + provider = new MockEmbeddingProvider(); + const tagged = await indexDocument(db, provider, { + title: "Tagged guide", + content: "Deploying the service with docker compose", + sourceType: "library", + library: "svc", + version: "2", + }); + addTagsToDocument(db, tagged.id, ["ops"]); + await indexDocument(db, provider, { + title: "Untagged guide", + content: "Deploying the service by hand", + sourceType: "manual", + }); + }); + + afterEach(() => { + db.close(); + }); + + it("returns the context prompt in passthrough mode without calling an LLM", async () => { + const complete = vi.fn(); + const llm: LlmProvider = { model: "m", complete }; + const result = await answer( + db, + provider, + { passthrough: true, llm }, + { question: "deploying" }, + ); + expect(result.mode).toBe("context"); + expect(complete).not.toHaveBeenCalled(); + if (result.mode === "context") expect(result.sources).toHaveLength(2); + }); + + it("asks the LLM and applies the filters to retrieval", async () => { + const complete = vi.fn((prompt: string) => Promise.resolve({ text: prompt, tokensUsed: 3 })); + const result = await answer( + db, + provider, + { passthrough: false, llm: { model: "m", complete } }, + { question: "deploying", tags: ["ops"], version: "2", sourceType: "library" }, + ); + expect(result.mode).toBe("answer"); + if (result.mode === "answer") { + expect(result.sources.map((s) => s.title)).toEqual(["Tagged guide"]); + expect(result.tokensUsed).toBe(3); + } + }); + + it("throws a ConfigError with setup instructions when no LLM is configured", async () => { + const result = answer(db, provider, { passthrough: false, llm: null }, { question: "q" }); + await expect(result).rejects.toBeInstanceOf(ConfigError); + await expect(result).rejects.toThrow("No LLM provider configured"); + }); +}); diff --git a/tests/unit/rag.test.ts b/tests/unit/rag.test.ts index d99a7e63..e097b96f 100644 --- a/tests/unit/rag.test.ts +++ b/tests/unit/rag.test.ts @@ -3,6 +3,8 @@ import { buildContextPrompt, extractSources, createLlmProvider, + isPassthroughMode, + resolveLlmProviderName, DEFAULT_SYSTEM_PROMPT, type LlmProvider, } from "../../src/core/rag.js"; @@ -216,7 +218,8 @@ describe("createLlmProvider", () => { it("creates an OpenAI provider when configured", () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test", model: "gpt-4o" }, + llm: { provider: "openai", model: "gpt-4o" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -255,7 +258,8 @@ describe("createLlmProvider", () => { it("uses default model for OpenAI when not specified", () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, + llm: { provider: "openai" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -265,10 +269,11 @@ describe("createLlmProvider", () => { expect(provider.model).toBe("gpt-4o-mini"); }); - it("falls back to embedding API key for OpenAI provider", () => { + it("uses the shared OpenAI key for embeddings and the LLM", () => { const config: LibScopeConfig = { - embedding: { provider: "openai", openaiApiKey: "sk-embed" }, + embedding: { provider: "openai" }, llm: { provider: "openai" }, + openai: { apiKey: "sk-shared" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -399,7 +404,8 @@ describe("createLlmProvider", () => { it("handles OpenAI empty choices response", async () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, + llm: { provider: "openai" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -422,7 +428,8 @@ describe("createLlmProvider", () => { it("handles OpenAI unknown status codes", async () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, + llm: { provider: "openai" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -445,7 +452,8 @@ describe("createLlmProvider", () => { it("handles OpenAI 429 rate limit", async () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", openaiApiKey: "sk-test" }, + llm: { provider: "openai" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -480,7 +488,8 @@ describe("createLlmProvider", () => { it("sanitizes OpenAI error messages (does not leak response body)", async () => { const config: LibScopeConfig = { embedding: { provider: "local" }, - llm: { provider: "openai", model: "gpt-4o", openaiApiKey: "sk-test" }, + llm: { provider: "openai", model: "gpt-4o" }, + openai: { apiKey: "sk-test" }, database: { path: ":memory:" }, indexing: { maxDocumentSize: 1024 }, logging: { level: "silent" }, @@ -595,3 +604,104 @@ describe("askQuestionStream", () => { } }); }); + +describe('llm.provider "auto"', () => { + function autoConfig(overrides: Partial = {}): LibScopeConfig { + return { + embedding: { provider: "local" }, + llm: { provider: "auto" }, + database: {}, + indexing: { maxDocumentSize: 1024, allowPrivateUrls: false, allowSelfSignedCerts: false }, + logging: { level: "silent" }, + ...overrides, + }; + } + + it("resolves to passthrough under MCP, even when API keys are set", () => { + const config = autoConfig({ openai: { apiKey: "sk-x" } }); + expect(resolveLlmProviderName(config, { surface: "mcp" })).toBe("passthrough"); + expect(isPassthroughMode(config, { surface: "mcp" })).toBe(true); + expect(() => createLlmProvider(config, { surface: "mcp" })).toThrow(/passthrough/); + }); + + it.each(["cli", "api", "sdk", undefined] as const)( + "is not passthrough on surface %s", + (surface) => { + expect(isPassthroughMode(autoConfig(), { surface })).toBe(false); + }, + ); + + it("picks openai when an OpenAI key is set", () => { + const config = autoConfig({ + openai: { apiKey: "sk-x" }, + anthropic: { apiKey: "sk-ant" }, + }); + expect(resolveLlmProviderName(config, { surface: "cli" })).toBe("openai"); + expect(createLlmProvider(config).model).toBe("gpt-4o-mini"); + }); + + it("picks anthropic when only an Anthropic key is set", () => { + const config = autoConfig({ anthropic: { apiKey: "sk-ant" } }); + expect(resolveLlmProviderName(config)).toBe("anthropic"); + expect(createLlmProvider(config).model).toBe("claude-3-5-haiku-20241022"); + }); + + it("picks ollama when llm.url is set", () => { + const config = autoConfig({ llm: { provider: "auto", url: "http://gpu:11434" } }); + expect(resolveLlmProviderName(config)).toBe("ollama"); + expect(createLlmProvider(config).model).toBe("llama3.2"); + }); + + it("picks ollama when the embedding provider is ollama", () => { + const config = autoConfig({ embedding: { provider: "ollama" } }); + expect(resolveLlmProviderName(config)).toBe("ollama"); + }); + + it("returns null and createLlmProvider throws a hint when nothing is configured", () => { + const config = autoConfig(); + expect(resolveLlmProviderName(config)).toBeNull(); + expect(() => createLlmProvider(config)).toThrow(/No LLM provider configured.*OPENAI_API_KEY/); + }); + + it("treats a missing llm section as auto", () => { + const config = autoConfig({ openai: { apiKey: "sk-x" } }); + delete config.llm; + expect(resolveLlmProviderName(config)).toBe("openai"); + }); + + it("an explicit provider is used on every surface", () => { + const config = autoConfig({ llm: { provider: "ollama" }, openai: { apiKey: "sk-x" } }); + expect(resolveLlmProviderName(config, { surface: "mcp" })).toBe("ollama"); + expect(isPassthroughMode(autoConfig({ llm: { provider: "passthrough" } }))).toBe(true); + }); + + it("the ollama LLM uses llm.url, then embedding.url", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ response: "hi" }), + }); + vi.stubGlobal("fetch", fetchMock); + try { + const fromEmbedding = createLlmProvider( + autoConfig({ + embedding: { provider: "ollama", url: "http://embed:11434" }, + llm: { provider: "ollama" }, + }), + ); + await fromEmbedding.complete("q"); + const fromLlm = createLlmProvider( + autoConfig({ + embedding: { provider: "ollama", url: "http://embed:11434" }, + llm: { provider: "ollama", url: "http://llm:11434" }, + }), + ); + await fromLlm.complete("q"); + expect(fetchMock.mock.calls.map((c) => String(c[0]))).toEqual([ + "http://embed:11434/api/generate", + "http://llm:11434/api/generate", + ]); + } finally { + vi.unstubAllGlobals(); + } + }); +}); diff --git a/tests/unit/registry-url-credentials.test.ts b/tests/unit/registry-url-credentials.test.ts new file mode 100644 index 00000000..034f45d1 --- /dev/null +++ b/tests/unit/registry-url-credentials.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { deriveRegistryName, validateGitUrl } from "../../src/registry/config.js"; + +describe("validateGitUrl credential check", () => { + it.each([ + "https://user:pass@github.com/org/repo.git", + "https://user:@github.com/org/repo.git", + "https://token@github.com/org/repo.git", + "https://github.com/org/repo?mirror=http://user@host", + "https://github.com/x/http://a:b@host", + ])("rejects %s", (url) => { + expect(() => validateGitUrl(url)).toThrow(/embedded credentials/); + }); + + it.each([ + "https://github.com/org/repo.git", + "https://github.com/org/repo@v1", + "https://@github.com/org/repo.git", + "ssh://git@host.example.com:7999/org/repo.git", + "git@github.com:org/repo.git", + "https://github.com/org/ftp://user@host", + ])("accepts %s", (url) => { + expect(() => validateGitUrl(url)).not.toThrow(); + }); + + it("checks a long URL quickly", () => { + const url = "https://" + "https://a".repeat(50_000); + const started = Date.now(); + expect(validateGitUrl(url)).toBe(url); + expect(Date.now() - started).toBeLessThan(1000); + }); + + it("removes every trailing slash", () => { + expect(validateGitUrl("https://github.com/org/repo.git///")).toBe( + "https://github.com/org/repo.git", + ); + }); +}); + +describe("deriveRegistryName", () => { + it("ignores trailing slashes", () => { + expect(deriveRegistryName("https://github.com/org/packs.git//")).toBe("packs"); + }); +}); diff --git a/tests/unit/registry/config.test.ts b/tests/unit/registry/config.test.ts index 98e89022..54cad00c 100644 --- a/tests/unit/registry/config.test.ts +++ b/tests/unit/registry/config.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { initLogger } from "../../../src/logger.js"; +import { NotFoundError } from "../../../src/errors.js"; import type { RegistryEntry } from "../../../src/registry/types.js"; // Mock homedir before importing registry config module @@ -28,14 +29,14 @@ const { validateRegistryName, validateGitUrl, sanitizeUrl, + requireRegistry, + deriveRegistryName, } = await import("../../../src/registry/config.js"); function makeEntry(overrides: Partial = {}): RegistryEntry { return { name: "test-reg", url: "https://github.com/org/registry.git", - syncInterval: 3600, - priority: 1, lastSyncedAt: null, ...overrides, }; @@ -110,20 +111,30 @@ describe("registry config", () => { expect(() => validateGitUrl("not-a-url")).toThrow(); }); - // Security/robustness fixes: whitespace trimming and trailing slash normalisation - it("should trim leading and trailing whitespace", () => { - const result = validateGitUrl(" https://github.com/org/repo.git "); - expect(result).toBe("https://github.com/org/repo.git"); - }); - - it("should strip trailing slashes", () => { - const result = validateGitUrl("https://github.com/org/repo/"); - expect(result).toBe("https://github.com/org/repo"); + it("accepts file:/// URLs and rejects plain local paths", () => { + expect(validateGitUrl("file:///srv/registry.git/")).toBe("file:///srv/registry.git"); + expect(() => validateGitUrl("/srv/registry.git")).toThrow(); }); - it("should strip both whitespace and trailing slashes together", () => { - const result = validateGitUrl(" https://github.com/org/repo.git "); - expect(result).toBe("https://github.com/org/repo.git"); + // Security/robustness fixes: whitespace trimming and trailing slash normalisation + it.each([ + { + name: "trims leading and trailing whitespace", + input: " https://github.com/org/repo.git ", + expected: "https://github.com/org/repo.git", + }, + { + name: "strips trailing slashes", + input: "https://github.com/org/repo/", + expected: "https://github.com/org/repo", + }, + { + name: "strips both whitespace and trailing slashes together", + input: " https://github.com/org/repo/ ", + expected: "https://github.com/org/repo", + }, + ])("should normalise the URL: $name", ({ input, expected }) => { + expect(validateGitUrl(input)).toBe(expected); }); // Security: reject embedded credentials @@ -284,8 +295,33 @@ describe("registry config", () => { expect(loadRegistries()).toHaveLength(0); }); - it("should throw when removing non-existent registry", () => { - expect(() => removeRegistry("nonexistent")).toThrow(/not found/); + it("should throw NotFoundError when removing non-existent registry", () => { + expect(() => removeRegistry("nonexistent")).toThrow(NotFoundError); + }); + }); + + describe("requireRegistry", () => { + it("returns the entry, or throws NotFoundError", () => { + addRegistry(makeEntry({ name: "here" })); + expect(requireRegistry("here").name).toBe("here"); + expect(() => requireRegistry("gone")).toThrow(NotFoundError); + }); + }); + + describe("deriveRegistryName", () => { + it.each([ + ["https://github.com/org/packs.git", "packs"], + ["https://github.com/org/packs/", "packs"], + ["git@github.com:org/team-packs.git", "team-packs"], + ["git@host:packs.git", "packs"], + ["ssh://git@host:7999/proj/my_packs.git", "my_packs"], + ["file:///srv/registries/local.git", "local"], + ])("%s -> %s", (url, name) => { + expect(deriveRegistryName(url)).toBe(name); + }); + + it("throws ValidationError when no valid name can be derived", () => { + expect(() => deriveRegistryName("https://example.com/x.git")).toThrow(/Give a name/); }); }); diff --git a/tests/unit/registry/conflict.test.ts b/tests/unit/registry/conflict.test.ts deleted file mode 100644 index 2838bf95..00000000 --- a/tests/unit/registry/conflict.test.ts +++ /dev/null @@ -1,449 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdirSync, writeFileSync, rmSync } from "node:fs"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; -import { randomUUID, createHash } from "node:crypto"; -import { initLogger } from "../../../src/logger.js"; -import type { RegistryEntry, PackSummary } from "../../../src/registry/types.js"; - -let tempHome: string = join(tmpdir(), `libscope-conflict-test-${process.pid}`); -mkdirSync(tempHome, { recursive: true }); - -vi.mock("node:os", async (importOriginal) => { - const orig = await importOriginal(); - return { - ...orig, - homedir: () => tempHome, - }; -}); - -const { - findPackInRegistries, - resolvePackFromRegistries, - parsePackSpecifier, - verifyResolvedPackChecksum, -} = await import("../../../src/registry/resolve.js"); -const { saveRegistries } = await import("../../../src/registry/config.js"); -const { getRegistryCacheDir, getPackDataPath, getPackManifestPath } = - await import("../../../src/registry/types.js"); -const { clearIndexCache } = await import("../../../src/registry/git.js"); - -function makeEntry(name: string, overrides: Partial = {}): RegistryEntry { - return { - name, - url: "https://github.com/org/registry.git", - syncInterval: 3600, - priority: 1, - lastSyncedAt: new Date().toISOString(), - ...overrides, - }; -} - -function makePack(name: string, overrides: Partial = {}): PackSummary { - return { - name, - description: `The ${name} pack`, - tags: [], - latestVersion: "1.0.0", - author: "author", - updatedAt: "2026-01-01T00:00:00.000Z", - ...overrides, - }; -} - -function setupRegistry(regName: string, packs: PackSummary[]): void { - const cacheDir = getRegistryCacheDir(regName); - mkdirSync(cacheDir, { recursive: true }); - writeFileSync(join(cacheDir, "index.json"), JSON.stringify(packs), "utf-8"); -} - -function setupPackDataFile(regName: string, packName: string, version: string): void { - const dataPath = getPackDataPath(regName, packName, version); - mkdirSync(join(dataPath, ".."), { recursive: true }); - writeFileSync( - dataPath, - JSON.stringify({ - name: packName, - version, - description: "test", - documents: [], - metadata: { author: "test", license: "MIT", createdAt: "2026-01-01" }, - }), - "utf-8", - ); -} - -/** - * Set up a pack data file with a matching manifest that includes a real checksum. - * Returns the checksum that was written. - */ -function setupPackDataFileWithManifest( - regName: string, - packName: string, - version: string, - content?: string, -): string { - const dataPath = getPackDataPath(regName, packName, version); - mkdirSync(join(dataPath, ".."), { recursive: true }); - - const packContent = - content ?? - JSON.stringify({ - name: packName, - version, - description: "test", - documents: [], - metadata: { author: "test", license: "MIT", createdAt: "2026-01-01" }, - }); - writeFileSync(dataPath, packContent, "utf-8"); - - const checksum = createHash("sha256").update(packContent, "utf-8").digest("hex"); - - const manifestPath = getPackManifestPath(regName, packName); - mkdirSync(join(manifestPath, ".."), { recursive: true }); - writeFileSync( - manifestPath, - JSON.stringify({ - name: packName, - description: "test", - tags: [], - author: "test", - license: "MIT", - versions: [ - { - version, - publishedAt: "2026-01-01T00:00:00.000Z", - checksumPath: `${version}/checksum.sha256`, - checksum, - docCount: 0, - }, - ], - }), - "utf-8", - ); - - return checksum; -} - -describe("registry conflict resolution", () => { - beforeEach(() => { - initLogger("silent"); - clearIndexCache(); - tempHome = join(tmpdir(), `libscope-conflict-${randomUUID()}`); - mkdirSync(tempHome, { recursive: true }); - }); - - afterEach(() => { - rmSync(tempHome, { recursive: true, force: true }); - }); - - describe("parsePackSpecifier", () => { - it("should parse 'name' as name only", () => { - expect(parsePackSpecifier("react-docs")).toEqual({ name: "react-docs" }); - }); - - it("should parse 'name@version' into name and version", () => { - expect(parsePackSpecifier("react-docs@1.2.0")).toEqual({ - name: "react-docs", - version: "1.2.0", - }); - }); - - it("should handle scoped-like names with @ at the start", () => { - // Last @ is the version delimiter - expect(parsePackSpecifier("@org/pack@2.0.0")).toEqual({ - name: "@org/pack", - version: "2.0.0", - }); - }); - - it("should return just name when no @ after first character", () => { - expect(parsePackSpecifier("simple-pack")).toEqual({ name: "simple-pack" }); - }); - }); - - describe("findPackInRegistries", () => { - it("should return empty matches when no registries configured", () => { - const { matches } = findPackInRegistries("anything"); - expect(matches).toEqual([]); - }); - - it("should find pack in single registry", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("react-docs")]); - - const { matches } = findPackInRegistries("react-docs"); - expect(matches).toHaveLength(1); - expect(matches[0]!.entry.name).toBe("reg1"); - expect(matches[0]!.pack.name).toBe("react-docs"); - }); - - it("should find pack in multiple registries", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - - const { matches } = findPackInRegistries("shared-pack"); - expect(matches).toHaveLength(2); - }); - - it("should warn for unsynced registries", () => { - saveRegistries([makeEntry("unsynced")]); - // No cache dir created - - const { matches, warnings } = findPackInRegistries("anything"); - expect(matches).toEqual([]); - expect(warnings).toHaveLength(1); - expect(warnings[0]).toContain("never been synced"); - }); - - it("should return empty matches when pack not found", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("other-pack")]); - - const { matches } = findPackInRegistries("nonexistent"); - expect(matches).toEqual([]); - }); - }); - - describe("resolvePackFromRegistries", () => { - it("should resolve pack from single registry", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("test-pack")]); - setupPackDataFile("reg1", "test-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("test-pack"); - expect(resolved).not.toBeNull(); - expect(resolved!.registryName).toBe("reg1"); - expect(resolved!.packName).toBe("test-pack"); - expect(resolved!.version).toBe("1.0.0"); - }); - - it("should return null when pack not found anywhere", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("other-pack")]); - - const { resolved } = resolvePackFromRegistries("nonexistent"); - expect(resolved).toBeNull(); - }); - - it("should detect conflict when pack exists in multiple registries", () => { - saveRegistries([makeEntry("reg1", { priority: 2 }), makeEntry("reg2", { priority: 1 })]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - setupPackDataFile("reg2", "shared-pack", "1.0.0"); - - // Default resolution is "priority" — reg2 has lower priority (wins) - const { resolved } = resolvePackFromRegistries("shared-pack"); - expect(resolved).not.toBeNull(); - expect(resolved!.registryName).toBe("reg2"); - }); - - it("should resolve conflict by priority (lower wins)", () => { - saveRegistries([makeEntry("reg-a", { priority: 10 }), makeEntry("reg-b", { priority: 1 })]); - setupRegistry("reg-a", [makePack("shared-pack")]); - setupRegistry("reg-b", [makePack("shared-pack")]); - setupPackDataFile("reg-b", "shared-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - conflictResolution: { strategy: "priority" }, - }); - expect(resolved!.registryName).toBe("reg-b"); - }); - - it("should resolve conflict with explicit registry", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - setupPackDataFile("reg1", "shared-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - conflictResolution: { strategy: "explicit", registryName: "reg1" }, - }); - expect(resolved!.registryName).toBe("reg1"); - }); - - it("should return conflict for interactive strategy without resolving", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - - const { resolved, conflict } = resolvePackFromRegistries("shared-pack", { - conflictResolution: { strategy: "interactive" }, - }); - expect(resolved).toBeNull(); - expect(conflict).toBeDefined(); - expect(conflict!.sources).toHaveLength(2); - expect(conflict!.packName).toBe("shared-pack"); - }); - - it("should return null when explicit registry doesn't have the pack", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - registryName: "reg3", - }); - expect(resolved).toBeNull(); - }); - - it("should filter to specified registryName option", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("shared-pack")]); - setupRegistry("reg2", [makePack("shared-pack")]); - setupPackDataFile("reg1", "shared-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("shared-pack", { - registryName: "reg1", - }); - expect(resolved!.registryName).toBe("reg1"); - }); - - it("should use specified version", () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("test-pack", { latestVersion: "2.0.0" })]); - setupPackDataFile("reg1", "test-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("test-pack", { version: "1.0.0" }); - expect(resolved!.version).toBe("1.0.0"); - }); - - it("should include all candidate registries in conflict", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2"), makeEntry("reg3")]); - setupRegistry("reg1", [makePack("shared")]); - setupRegistry("reg2", [makePack("shared")]); - setupRegistry("reg3", [makePack("shared")]); - - const { conflict } = resolvePackFromRegistries("shared", { - conflictResolution: { strategy: "interactive" }, - }); - expect(conflict!.sources).toHaveLength(3); - }); - - it("should not conflict when packs have different names", () => { - saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); - setupRegistry("reg1", [makePack("pack-a")]); - setupRegistry("reg2", [makePack("pack-b")]); - setupPackDataFile("reg1", "pack-a", "1.0.0"); - - const { resolved, conflict } = resolvePackFromRegistries("pack-a"); - expect(resolved).not.toBeNull(); - expect(conflict).toBeUndefined(); - }); - }); - - describe("verifyResolvedPackChecksum", () => { - it("should pass when file matches recorded checksum", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("test-pack")]); - setupPackDataFileWithManifest("reg1", "test-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("test-pack"); - expect(resolved).not.toBeNull(); - - await expect(verifyResolvedPackChecksum(resolved!)).resolves.toBeUndefined(); - }); - - it("should throw when file has been tampered with", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("test-pack")]); - setupPackDataFileWithManifest("reg1", "test-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("test-pack"); - expect(resolved).not.toBeNull(); - - // Tamper with the pack data file after checksum was recorded - writeFileSync(resolved!.dataPath, '{"tampered":true}', "utf-8"); - - await expect(verifyResolvedPackChecksum(resolved!)).rejects.toThrow( - /Checksum verification failed/, - ); - }); - - it("should skip verification and warn when no manifest exists", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("no-manifest-pack")]); - setupPackDataFile("reg1", "no-manifest-pack", "1.0.0"); - - const { resolved } = resolvePackFromRegistries("no-manifest-pack"); - expect(resolved).not.toBeNull(); - - // Should not throw — manifest is missing, logs a warning and skips - await expect(verifyResolvedPackChecksum(resolved!)).resolves.toBeUndefined(); - }); - - it("should throw when manifest has no checksum for the version", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("zero-checksum-pack")]); - setupPackDataFile("reg1", "zero-checksum-pack", "1.0.0"); - - // Write a manifest with an empty checksum - const manifestPath = getPackManifestPath("reg1", "zero-checksum-pack"); - mkdirSync(join(manifestPath, ".."), { recursive: true }); - writeFileSync( - manifestPath, - JSON.stringify({ - name: "zero-checksum-pack", - description: "test", - tags: [], - author: "test", - license: "MIT", - versions: [ - { - version: "1.0.0", - publishedAt: "2026-01-01T00:00:00.000Z", - checksumPath: "1.0.0/checksum.sha256", - checksum: "", - docCount: 0, - }, - ], - }), - "utf-8", - ); - - const { resolved } = resolvePackFromRegistries("zero-checksum-pack"); - expect(resolved).not.toBeNull(); - - await expect(verifyResolvedPackChecksum(resolved!)).rejects.toThrow( - /has no checksum recorded/, - ); - }); - - it("should skip verification when version entry not found in manifest", async () => { - saveRegistries([makeEntry("reg1")]); - setupRegistry("reg1", [makePack("version-mismatch-pack")]); - setupPackDataFile("reg1", "version-mismatch-pack", "1.0.0"); - - // Manifest only records version 2.0.0, not 1.0.0 - const manifestPath = getPackManifestPath("reg1", "version-mismatch-pack"); - mkdirSync(join(manifestPath, ".."), { recursive: true }); - writeFileSync( - manifestPath, - JSON.stringify({ - name: "version-mismatch-pack", - description: "test", - tags: [], - author: "test", - license: "MIT", - versions: [ - { - version: "2.0.0", - publishedAt: "2026-01-01T00:00:00.000Z", - checksumPath: "2.0.0/checksum.sha256", - checksum: "abc123", - docCount: 0, - }, - ], - }), - "utf-8", - ); - - const { resolved } = resolvePackFromRegistries("version-mismatch-pack"); - expect(resolved).not.toBeNull(); - - // Should not throw — version entry missing, logs a warning and skips - await expect(verifyResolvedPackChecksum(resolved!)).resolves.toBeUndefined(); - }); - }); -}); diff --git a/tests/unit/registry/git.test.ts b/tests/unit/registry/git.test.ts index 035b5bf7..fdf218b4 100644 --- a/tests/unit/registry/git.test.ts +++ b/tests/unit/registry/git.test.ts @@ -16,12 +16,13 @@ const gitEnv = { import { readIndex, createRegistryRepo, - checkGitAvailable, + git, cloneRegistry, fetchRegistry, commitAndPush, clearIndexCache, } from "../../../src/registry/git.js"; +import { ConfigError } from "../../../src/errors.js"; describe("registry git helpers", () => { let tempDir: string; @@ -35,10 +36,15 @@ describe("registry git helpers", () => { rmSync(tempDir, { recursive: true, force: true }); }); - describe("checkGitAvailable", () => { - it("should return true when git is available", async () => { - const result = await checkGitAvailable(); - expect(result).toBe(true); + describe("git", () => { + it("reports a missing git binary as a ConfigError", async () => { + const savedPath = process.env["PATH"]; + process.env["PATH"] = tempDir; + try { + await expect(git(["--version"])).rejects.toBeInstanceOf(ConfigError); + } finally { + process.env["PATH"] = savedPath; + } }); }); @@ -221,6 +227,27 @@ describe("registry git helpers", () => { const log = execSync("git log --oneline", { cwd: repoPath, encoding: "utf-8" }); expect(log).toContain("Initial registry structure"); }); + + it("should commit on main even when git's default branch is master", async () => { + const repoPath = join(tempDir, "main-registry"); + const saved = { ...process.env }; + Object.assign(process.env, { + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "init.defaultBranch", + GIT_CONFIG_VALUE_0: "master", + }); + try { + await createRegistryRepo(repoPath); + } finally { + process.env = saved; + } + + const branch = execSync("git rev-parse --abbrev-ref HEAD", { + cwd: repoPath, + encoding: "utf-8", + }); + expect(branch.trim()).toBe("main"); + }); }); describe("cloneRegistry + fetchRegistry", () => { @@ -341,6 +368,30 @@ describe("registry git helpers", () => { const content = readFileSync(join(verifyDir, "file.txt"), "utf-8"); expect(content).toBe("updated"); }); + + it("should push a new branch that has no upstream yet", async () => { + const bareDir = join(tempDir, "branch-bare.git"); + execSync(`git init --bare "${bareDir}"`, { stdio: "pipe" }); + const workDir = join(tempDir, "branch-work"); + execSync(`git clone "${bareDir}" "${workDir}"`, { stdio: "pipe" }); + writeFileSync(join(workDir, "file.txt"), "initial", "utf-8"); + execSync("git add . && git commit -m 'init'", { cwd: workDir, stdio: "pipe", env: gitEnv }); + execSync("git push origin HEAD", { cwd: workDir, stdio: "pipe" }); + + execSync("git checkout -b feature/add-pack", { cwd: workDir, stdio: "pipe" }); + writeFileSync(join(workDir, "file.txt"), "on branch", "utf-8"); + const saved = { ...process.env }; + // Ignore the user's git config (push.autoSetupRemote would hide the missing upstream) + Object.assign(process.env, { GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }); + try { + await commitAndPush(workDir, "branch commit"); + } finally { + process.env = saved; + } + + const branches = execSync("git branch --list", { cwd: bareDir, encoding: "utf-8" }); + expect(branches).toContain("feature/add-pack"); + }); }); // Robustness: LIBSCOPE_GIT_TIMEOUT_MS controls git timeout diff --git a/tests/unit/registry/publish.test.ts b/tests/unit/registry/publish.test.ts index aeff7240..8cfefa99 100644 --- a/tests/unit/registry/publish.test.ts +++ b/tests/unit/registry/publish.test.ts @@ -64,8 +64,6 @@ function makeEntry( return { name, url, - syncInterval: 3600, - priority: 1, lastSyncedAt: null, ...overrides, }; @@ -225,42 +223,20 @@ describe("registry publish", () => { // ---------------------------------------------------------------- describe("semver version validation", () => { - it("should reject a version string that is not semver ('abc')", async () => { + it.each([ + { label: "a version string that is not semver", version: "abc", file: "bad-ver.json" }, + { label: "a version with only two parts", version: "1.2", file: "two-part.json" }, + { label: "a version with four parts", version: "1.2.3.4", file: "four-part.json" }, + ])("should reject $label ('$version')", async ({ version, file }) => { const regName = `reg-${randomUUID().slice(0, 8)}`; const { bareUrl } = createRegistryCache(tempDir, regName); addTestRegistry(makeEntry(regName, bareUrl)); - const packFile = join(tempDir, "bad-ver.json"); - writeFileSync(packFile, JSON.stringify(makePackJson("good-pack", "abc")), "utf-8"); + const packFile = join(tempDir, file); + writeFileSync(packFile, JSON.stringify(makePackJson("good-pack", version)), "utf-8"); await expect( - publishPack({ registryName: regName, packFilePath: packFile, version: "abc" }), - ).rejects.toThrow(/semver/i); - }); - - it("should reject a version with only two parts ('1.2')", async () => { - const regName = `reg-${randomUUID().slice(0, 8)}`; - const { bareUrl } = createRegistryCache(tempDir, regName); - addTestRegistry(makeEntry(regName, bareUrl)); - - const packFile = join(tempDir, "two-part.json"); - writeFileSync(packFile, JSON.stringify(makePackJson("good-pack", "1.2")), "utf-8"); - - await expect( - publishPack({ registryName: regName, packFilePath: packFile, version: "1.2" }), - ).rejects.toThrow(/semver/i); - }); - - it("should reject a version with four parts ('1.2.3.4')", async () => { - const regName = `reg-${randomUUID().slice(0, 8)}`; - const { bareUrl } = createRegistryCache(tempDir, regName); - addTestRegistry(makeEntry(regName, bareUrl)); - - const packFile = join(tempDir, "four-part.json"); - writeFileSync(packFile, JSON.stringify(makePackJson("good-pack", "1.2.3.4")), "utf-8"); - - await expect( - publishPack({ registryName: regName, packFilePath: packFile, version: "1.2.3.4" }), + publishPack({ registryName: regName, packFilePath: packFile, version }), ).rejects.toThrow(/semver/i); }); diff --git a/tests/unit/registry/resolve.test.ts b/tests/unit/registry/resolve.test.ts new file mode 100644 index 00000000..3476bc03 --- /dev/null +++ b/tests/unit/registry/resolve.test.ts @@ -0,0 +1,235 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { gzipSync } from "node:zlib"; +import { randomUUID, createHash } from "node:crypto"; +import { initLogger } from "../../../src/logger.js"; +import { NotFoundError, ValidationError } from "../../../src/errors.js"; +import type { RegistryEntry, PackSummary } from "../../../src/registry/types.js"; + +let tempHome: string = join(tmpdir(), `libscope-resolve-test-${process.pid}`); + +vi.mock("node:os", async (importOriginal) => { + const orig = await importOriginal(); + return { ...orig, homedir: (): string => tempHome }; +}); + +const { findRegistryPack, resolveRegistryPack, parsePackSpecifier, verifyResolvedPackChecksum } = + await import("../../../src/registry/resolve.js"); +const { saveRegistries } = await import("../../../src/registry/config.js"); +const { getRegistryCacheDir, getPackDataPath, getPackManifestPath } = + await import("../../../src/registry/types.js"); +const { clearIndexCache } = await import("../../../src/registry/git.js"); + +function makeEntry(name: string): RegistryEntry { + return { name, url: "https://github.com/org/registry.git", lastSyncedAt: null }; +} + +function makePack(name: string, overrides: Partial = {}): PackSummary { + return { + name, + description: `The ${name} pack`, + tags: [], + latestVersion: "1.0.0", + author: "author", + updatedAt: "2026-01-01T00:00:00.000Z", + ...overrides, + }; +} + +function setupRegistry(regName: string, packs: PackSummary[]): void { + const cacheDir = getRegistryCacheDir(regName); + mkdirSync(cacheDir, { recursive: true }); + writeFileSync(join(cacheDir, "index.json"), JSON.stringify(packs), "utf-8"); +} + +function packJson(name: string, version: string): string { + return JSON.stringify({ + name, + version, + description: "test", + documents: [], + metadata: { author: "test", license: "MIT", createdAt: "2026-01-01" }, + }); +} + +/** Write the pack data file and, unless `checksum` is null, a manifest recording it. */ +function setupPackData( + regName: string, + packName: string, + version: string, + options: { data?: Buffer | string; checksum?: string | null; manifestVersion?: string } = {}, +): string { + const dataPath = getPackDataPath(regName, packName, version); + mkdirSync(join(dataPath, ".."), { recursive: true }); + const data = options.data ?? packJson(packName, version); + writeFileSync(dataPath, data); + if (options.checksum === null) return dataPath; + + const manifestPath = getPackManifestPath(regName, packName); + mkdirSync(join(manifestPath, ".."), { recursive: true }); + const checksum = options.checksum ?? createHash("sha256").update(data).digest("hex"); + const entryVersion = options.manifestVersion ?? version; + writeFileSync( + manifestPath, + JSON.stringify({ + name: packName, + description: "test", + tags: [], + author: "test", + license: "MIT", + versions: [ + { + version: entryVersion, + publishedAt: "2026-01-01T00:00:00.000Z", + checksumPath: `${entryVersion}/checksum.sha256`, + checksum, + docCount: 0, + }, + ], + }), + ); + return dataPath; +} + +describe("registry pack resolution", () => { + beforeEach(() => { + initLogger("silent"); + clearIndexCache(); + tempHome = join(tmpdir(), `libscope-resolve-${randomUUID()}`); + mkdirSync(tempHome, { recursive: true }); + }); + + afterEach(() => { + rmSync(tempHome, { recursive: true, force: true }); + }); + + describe("parsePackSpecifier", () => { + it("parses 'name' and 'name@version'", () => { + expect(parsePackSpecifier("react-docs")).toEqual({ name: "react-docs" }); + expect(parsePackSpecifier("react-docs@1.2.0")).toEqual({ + name: "react-docs", + version: "1.2.0", + }); + }); + + it("does not treat a leading @ as a version separator", () => { + expect(parsePackSpecifier("@scope")).toEqual({ name: "@scope" }); + }); + }); + + describe("findRegistryPack", () => { + it("finds the latest version in the single registry that has the pack", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("react", { latestVersion: "2.0.0" })]); + const dataPath = setupPackData("reg1", "react", "2.0.0"); + + expect(findRegistryPack("react")).toEqual({ + registry: "reg1", + name: "react", + version: "2.0.0", + dataPath, + }); + }); + + it("installs the version given as name@version", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("react", { latestVersion: "2.0.0" })]); + setupPackData("reg1", "react", "2.0.0"); + setupPackData("reg1", "react", "1.0.0"); + + expect(findRegistryPack("react@1.0.0").version).toBe("1.0.0"); + }); + + it("throws ValidationError naming every registry when several have the pack", () => { + saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); + setupRegistry("reg1", [makePack("react")]); + setupRegistry("reg2", [makePack("react")]); + + expect(() => findRegistryPack("react")).toThrow(ValidationError); + expect(() => findRegistryPack("react")).toThrow(/several registries \(reg1, reg2\)/); + }); + + it("uses the named registry when several have the pack", () => { + saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); + setupRegistry("reg1", [makePack("react")]); + setupRegistry("reg2", [makePack("react")]); + setupPackData("reg2", "react", "1.0.0"); + + expect(findRegistryPack("react", "reg2").registry).toBe("reg2"); + }); + + it("throws NotFoundError when no registry has the pack", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("other")]); + + expect(() => findRegistryPack("react")).toThrow(NotFoundError); + expect(() => findRegistryPack("react", "reg1")).toThrow(/not found in registry "reg1"/); + }); + + it("throws NotFoundError when no registries are configured", () => { + expect(() => findRegistryPack("react")).toThrow(/no pack registries are configured/); + }); + + it("throws NotFoundError for an unknown registry name", () => { + saveRegistries([makeEntry("reg1")]); + expect(() => findRegistryPack("react", "nope")).toThrow(NotFoundError); + }); + + it("throws NotFoundError when the version is not in the local copy", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("react")]); + setupPackData("reg1", "react", "1.0.0"); + + expect(() => findRegistryPack("react@9.9.9")).toThrow(NotFoundError); + }); + + it("mentions registries that were never synced", () => { + saveRegistries([makeEntry("reg1"), makeEntry("unsynced")]); + setupRegistry("reg1", []); + + expect(() => findRegistryPack("react")).toThrow(/"unsynced" has never been synced/); + }); + + it("rejects path traversal in names and versions", () => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("react", { latestVersion: "../../etc" })]); + + expect(() => findRegistryPack("../react")).toThrow(ValidationError); + expect(() => findRegistryPack("react@../x")).toThrow(ValidationError); + expect(() => findRegistryPack("react")).toThrow(ValidationError); + }); + }); + + describe("resolveRegistryPack (checksums)", () => { + beforeEach(() => { + saveRegistries([makeEntry("reg1")]); + setupRegistry("reg1", [makePack("p")]); + }); + + it("resolves when the file matches the recorded checksum, also for gzip data", async () => { + setupPackData("reg1", "p", "1.0.0", { data: gzipSync(packJson("p", "1.0.0")) }); + await expect(resolveRegistryPack("p")).resolves.toMatchObject({ name: "p" }); + }); + + it("throws when the file was changed after publishing", async () => { + const dataPath = setupPackData("reg1", "p", "1.0.0"); + writeFileSync(dataPath, '{"tampered":true}'); + await expect(resolveRegistryPack("p")).rejects.toThrow(/Checksum verification failed/); + }); + + it("throws when the manifest has no checksum for the version", async () => { + setupPackData("reg1", "p", "1.0.0", { checksum: "" }); + await expect(resolveRegistryPack("p")).rejects.toThrow(/has no checksum recorded/); + }); + + it("skips verification without a manifest or a manifest entry for the version", async () => { + setupPackData("reg1", "p", "1.0.0", { checksum: null }); + await expect(resolveRegistryPack("p")).resolves.toMatchObject({ version: "1.0.0" }); + + setupPackData("reg1", "p", "1.0.0", { manifestVersion: "2.0.0", checksum: "abc" }); + await expect(verifyResolvedPackChecksum(findRegistryPack("p"))).resolves.toBeUndefined(); + }); + }); +}); diff --git a/tests/unit/registry/search.test.ts b/tests/unit/registry/search.test.ts index d89da2a1..42c6507b 100644 --- a/tests/unit/registry/search.test.ts +++ b/tests/unit/registry/search.test.ts @@ -4,9 +4,10 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { initLogger } from "../../../src/logger.js"; +import { NotFoundError } from "../../../src/errors.js"; import type { RegistryEntry, PackSummary } from "../../../src/registry/types.js"; -// Mock homedir before importing any registry modules — REGISTRIES_DIR is module-level +// Registry paths follow homedir(): point it at a temp directory. let tempHome: string = join(tmpdir(), `libscope-search-test-${process.pid}`); mkdirSync(tempHome, { recursive: true }); @@ -19,7 +20,7 @@ vi.mock("node:os", async (importOriginal) => { }); // Import AFTER mock is set up — getRegistryCacheDir picks up mocked homedir -const { searchRegistries } = await import("../../../src/registry/search.js"); +const { searchRegistries, listRegistryPacks } = await import("../../../src/registry/search.js"); const { saveRegistries } = await import("../../../src/registry/config.js"); const { getRegistryCacheDir } = await import("../../../src/registry/types.js"); const { clearIndexCache } = await import("../../../src/registry/git.js"); @@ -28,8 +29,6 @@ function makeEntry(name: string, overrides: Partial = {}): Regist return { name, url: "https://github.com/org/registry.git", - syncInterval: 3600, - priority: 1, lastSyncedAt: new Date().toISOString(), ...overrides, }; @@ -88,7 +87,7 @@ describe("registry search", () => { const { results } = searchRegistries("react-docs"); expect(results).toHaveLength(1); - expect(results[0]!.pack.name).toBe("react-docs"); + expect(results[0]!.name).toBe("react-docs"); // 100 (exact name) + 20 (description contains "react-docs" via default desc) expect(results[0]!.score).toBeGreaterThanOrEqual(100); }); @@ -144,7 +143,7 @@ describe("registry search", () => { const { results } = searchRegistries("docs"); expect(results).toHaveLength(2); - expect(results.map((r) => r.pack.name).sort((a, b) => a.localeCompare(b))).toEqual([ + expect(results.map((r) => r.name).sort((a, b) => a.localeCompare(b))).toEqual([ "react-docs", "vue-docs", ]); @@ -160,7 +159,7 @@ describe("registry search", () => { const { results } = searchRegistries("react"); // "react" has exact name match (100) + more → higher score // "react-docs" has partial name match (50) + less - expect(results[0]!.pack.name).toBe("react"); + expect(results[0]!.name).toBe("react"); expect(results[0]!.score).toBeGreaterThan(results[1]!.score); }); @@ -179,14 +178,21 @@ describe("registry search", () => { const { results } = searchRegistries("docs", { registryName: "reg1" }); expect(results).toHaveLength(1); - expect(results[0]!.registryName).toBe("reg1"); + expect(results[0]!.registry).toBe("reg1"); }); - it("should warn when specified registryName does not exist", () => { - const { results, warnings } = searchRegistries("test", { registryName: "nonexistent" }); - expect(results).toEqual([]); - expect(warnings).toHaveLength(1); - expect(warnings[0]).toContain("not found"); + it("throws NotFoundError when the named registry does not exist", () => { + expect(() => searchRegistries("test", { registryName: "nonexistent" })).toThrow(NotFoundError); + }); + + it("lists every pack of every registry with listRegistryPacks", () => { + saveRegistries([makeEntry("reg1"), makeEntry("reg2")]); + setupRegistry("reg1", [makePack("a"), makePack("b")]); + setupRegistry("reg2", [makePack("c")]); + const { packs, warnings } = listRegistryPacks(); + expect(packs.map((p) => `${p.registry}/${p.name}`)).toEqual(["reg1/a", "reg1/b", "reg2/c"]); + expect(warnings).toEqual([]); + expect(listRegistryPacks("reg2").packs).toHaveLength(1); }); it("should include registryName in results", () => { @@ -194,7 +200,7 @@ describe("registry search", () => { setupRegistry("my-registry", [makePack("test-pack")]); const { results } = searchRegistries("test"); - expect(results[0]!.registryName).toBe("my-registry"); + expect(results[0]!.registry).toBe("my-registry"); }); it("should handle corrupted index.json gracefully", () => { diff --git a/tests/unit/registry/stale-cache.test.ts b/tests/unit/registry/stale-cache.test.ts deleted file mode 100644 index 834215ef..00000000 --- a/tests/unit/registry/stale-cache.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import type { RegistryEntry } from "../../../src/registry/types.js"; -import { isRegistryStale } from "../../../src/registry/sync.js"; - -function makeEntry(overrides: Partial = {}): RegistryEntry { - return { - name: "test-reg", - url: "https://github.com/org/registry.git", - syncInterval: 3600, // 1 hour - priority: 1, - lastSyncedAt: null, - ...overrides, - }; -} - -describe("registry stale-cache detection", () => { - beforeEach(() => { - vi.useFakeTimers(); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - describe("isRegistryStale", () => { - it("should return true when lastSyncedAt is null (never synced)", () => { - const entry = makeEntry({ lastSyncedAt: null }); - expect(isRegistryStale(entry)).toBe(true); - }); - - it("should return false when syncInterval is 0 (manual only)", () => { - const entry = makeEntry({ syncInterval: 0, lastSyncedAt: null }); - expect(isRegistryStale(entry)).toBe(false); - }); - - it("should return false when syncInterval is negative", () => { - const entry = makeEntry({ syncInterval: -1, lastSyncedAt: null }); - expect(isRegistryStale(entry)).toBe(false); - }); - - it("should return true when last sync was longer ago than syncInterval", () => { - const now = new Date("2026-03-11T12:00:00.000Z"); - vi.setSystemTime(now); - - // Last synced 2 hours ago, interval is 1 hour - const entry = makeEntry({ - syncInterval: 3600, - lastSyncedAt: "2026-03-11T10:00:00.000Z", - }); - expect(isRegistryStale(entry)).toBe(true); - }); - - it("should return false when last sync was within syncInterval", () => { - const now = new Date("2026-03-11T12:00:00.000Z"); - vi.setSystemTime(now); - - // Last synced 30 minutes ago, interval is 1 hour - const entry = makeEntry({ - syncInterval: 3600, - lastSyncedAt: "2026-03-11T11:30:00.000Z", - }); - expect(isRegistryStale(entry)).toBe(false); - }); - - it("should return true at exactly the boundary (1 ms past interval)", () => { - // syncInterval = 60 seconds = 60000ms - const entry = makeEntry({ syncInterval: 60 }); - const baseTime = new Date("2026-03-11T12:00:00.000Z"); - entry.lastSyncedAt = baseTime.toISOString(); - - // Set time to 60001ms later (1ms past the boundary) - vi.setSystemTime(new Date(baseTime.getTime() + 60001)); - expect(isRegistryStale(entry)).toBe(true); - }); - - it("should return false at exactly the boundary (exactly syncInterval)", () => { - const entry = makeEntry({ syncInterval: 60 }); - const baseTime = new Date("2026-03-11T12:00:00.000Z"); - entry.lastSyncedAt = baseTime.toISOString(); - - // Set time to exactly 60000ms later - vi.setSystemTime(new Date(baseTime.getTime() + 60000)); - expect(isRegistryStale(entry)).toBe(false); - }); - - it("should handle very short syncInterval (1 second)", () => { - const entry = makeEntry({ syncInterval: 1 }); - const baseTime = new Date("2026-03-11T12:00:00.000Z"); - entry.lastSyncedAt = baseTime.toISOString(); - - vi.setSystemTime(new Date(baseTime.getTime() + 2000)); - expect(isRegistryStale(entry)).toBe(true); - }); - - it("should handle very large syncInterval (24 hours)", () => { - const entry = makeEntry({ syncInterval: 86400 }); - const now = new Date("2026-03-11T12:00:00.000Z"); - vi.setSystemTime(now); - - // Synced 12 hours ago — still fresh - entry.lastSyncedAt = "2026-03-11T00:00:00.000Z"; - expect(isRegistryStale(entry)).toBe(false); - - // Synced 25 hours ago — stale - entry.lastSyncedAt = "2026-03-10T11:00:00.000Z"; - expect(isRegistryStale(entry)).toBe(true); - }); - }); -}); diff --git a/tests/unit/registry/sync.test.ts b/tests/unit/registry/sync.test.ts index 0d4edc50..2b9cd97d 100644 --- a/tests/unit/registry/sync.test.ts +++ b/tests/unit/registry/sync.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdirSync, writeFileSync, rmSync, mkdtempSync, existsSync } from "node:fs"; +import { mkdirSync, writeFileSync, rmSync, mkdtempSync, existsSync, renameSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; @@ -18,13 +18,8 @@ vi.mock("node:os", async (importOriginal) => { }; }); -const { - syncRegistry, - syncAllRegistries, - syncStaleRegistries, - syncRegistryByName, - getRegistryIndex, -} = await import("../../../src/registry/sync.js"); +const { syncRegistry, syncAllRegistries, localPackCount } = + await import("../../../src/registry/sync.js"); const { loadRegistries, saveRegistries } = await import("../../../src/registry/config.js"); function makeEntry( @@ -32,16 +27,18 @@ function makeEntry( url: string, overrides: Partial = {}, ): RegistryEntry { - return { - name, - url, - syncInterval: 3600, - priority: 1, - lastSyncedAt: null, - ...overrides, - }; + return { name, url, lastSyncedAt: null, ...overrides }; } +const PACK: PackSummary = { + name: "test-pack", + description: "Test", + tags: [], + latestVersion: "1.0.0", + author: "a", + updatedAt: "2026-01-01", +}; + function addTestRegistry(entry: RegistryEntry): void { const registries = loadRegistries(); registries.push(entry); @@ -80,18 +77,33 @@ describe("registry sync functions", () => { rmSync(tempDir, { recursive: true, force: true }); }); - describe("syncRegistryByName", () => { - it("should return error for non-existent registry name", async () => { - const status = await syncRegistryByName("nonexistent"); - expect(status.status).toBe("error"); - expect(status.error).toContain("not found"); + describe("syncRegistry", () => { + it("clones the registry and reports its pack count", async () => { + const bareRepo = createBareRepo(tempDir, [PACK]); + addTestRegistry(makeEntry("by-name", bareRepo)); + expect(localPackCount("by-name")).toBeNull(); + + const status = await syncRegistry(makeEntry("by-name", bareRepo)); + expect(status).toMatchObject({ registry: "by-name", status: "success", packs: 1 }); + expect(loadRegistries()[0]!.lastSyncedAt).not.toBeNull(); }); - it("should sync an existing registry by name", async () => { - const bareRepo = createBareRepo(tempDir); - addTestRegistry(makeEntry("by-name", bareRepo)); - const status = await syncRegistryByName("by-name"); - expect(status.status).toBe("success"); + it("keeps the local copy when the remote is unreachable (offline)", async () => { + const repo = createBareRepo(tempDir, [PACK]); + const entry = makeEntry("offline-idx", repo); + addTestRegistry(entry); + await syncRegistry(entry); + + renameSync(repo, repo + ".broken"); + const status = await syncRegistry(entry); + expect(status).toMatchObject({ status: "offline", packs: 1 }); + expect(status.error).toBeTruthy(); + }); + + it("reports error when the registry was never cloned and cannot be reached", async () => { + const entry = makeEntry("never", join(tempDir, "missing.git")); + addTestRegistry(entry); + expect(await syncRegistry(entry)).toMatchObject({ status: "error", packs: null }); }); }); @@ -113,102 +125,6 @@ describe("registry sync functions", () => { }); }); - describe("syncStaleRegistries", () => { - it("should return empty array when no registries are stale", async () => { - const repo = createBareRepo(tempDir); - addTestRegistry( - makeEntry("fresh", repo, { - syncInterval: 99999, - lastSyncedAt: new Date().toISOString(), - }), - ); - const results = await syncStaleRegistries(); - expect(results).toEqual([]); - }); - - it("should sync registries that are stale", async () => { - const repo = createBareRepo(tempDir); - addTestRegistry( - makeEntry("stale-one", repo, { - syncInterval: 1, - lastSyncedAt: "2020-01-01T00:00:00.000Z", // very old - }), - ); - const results = await syncStaleRegistries(); - expect(results).toHaveLength(1); - expect(results[0]!.status).toBe("success"); - }); - - it("should return empty when all registries have syncInterval=0 (manual)", async () => { - const repo = createBareRepo(tempDir); - addTestRegistry(makeEntry("manual", repo, { syncInterval: 0 })); - const results = await syncStaleRegistries(); - expect(results).toEqual([]); - }); - }); - - describe("getRegistryIndex", () => { - it("should return packs from a synced registry", async () => { - const packs: PackSummary[] = [ - { - name: "test-pack", - description: "Test", - tags: [], - latestVersion: "1.0.0", - author: "a", - updatedAt: "2026-01-01", - }, - ]; - const repo = createBareRepo(tempDir, packs); - addTestRegistry(makeEntry("idx-test", repo)); - await syncRegistry(makeEntry("idx-test", repo)); - - const entry = makeEntry("idx-test", repo, { - syncInterval: 0, // manual, won't auto-sync - lastSyncedAt: new Date().toISOString(), - }); - const { packs: result, warning } = await getRegistryIndex(entry); - expect(result).toHaveLength(1); - expect(result[0]!.name).toBe("test-pack"); - expect(warning).toBeUndefined(); - }); - - it("should return warning when remote unreachable and has stale cache", async () => { - const packs: PackSummary[] = [ - { - name: "offline-pack", - description: "Offline", - tags: [], - latestVersion: "1.0.0", - author: "a", - updatedAt: "2026-01-01", - }, - ]; - const repo = createBareRepo(tempDir, packs); - const entry = makeEntry("offline-idx", repo, { - syncInterval: 1, - lastSyncedAt: "2020-01-01T00:00:00.000Z", - }); - addTestRegistry(entry); - - // Sync once to populate cache - await syncRegistry(entry); - - // Break the remote - const { renameSync } = await import("node:fs"); - renameSync(repo, repo + ".broken"); - - // getRegistryIndex should fall back to cache with warning - const staleEntry = makeEntry("offline-idx", repo, { - syncInterval: 1, - lastSyncedAt: "2020-01-01T00:00:00.000Z", - }); - const { packs: result, warning } = await getRegistryIndex(staleEntry); - expect(result).toHaveLength(1); - expect(warning).toContain("unreachable"); - }); - }); - // Robustness fix: sync locking prevents concurrent syncs describe("sync locking", () => { it("should create a lock file during sync and remove it when done", async () => { diff --git a/tests/unit/registry/types.test.ts b/tests/unit/registry/types.test.ts index e5e0c6f9..8a081a37 100644 --- a/tests/unit/registry/types.test.ts +++ b/tests/unit/registry/types.test.ts @@ -15,8 +15,6 @@ import type { RegistryEntry, PackSummary, PackManifest, - RegistryConfigBlock, - ConflictResolution, RegistrySyncStatus, } from "../../../src/registry/types.js"; @@ -73,12 +71,9 @@ describe("registry types — type shape validation", () => { const entry: RegistryEntry = { name: "official", url: "https://github.com/org/registry.git", - syncInterval: 3600, - priority: 1, lastSyncedAt: null, }; expect(entry.name).toBe("official"); - expect(entry.syncInterval).toBe(3600); expect(entry.lastSyncedAt).toBeNull(); }); @@ -116,25 +111,9 @@ describe("registry types — type shape validation", () => { expect(manifest.versions[0]!.docCount).toBe(5); }); - it("RegistryConfigBlock should wrap registries array", () => { - const block: RegistryConfigBlock = { - registries: [], - }; - expect(block.registries).toEqual([]); - }); - - it("ConflictResolution should support 'priority', 'interactive', and 'explicit' strategies", () => { - const byPriority: ConflictResolution = { strategy: "priority" }; - const interactive: ConflictResolution = { strategy: "interactive" }; - const explicit: ConflictResolution = { strategy: "explicit", registryName: "official" }; - expect(byPriority.strategy).toBe("priority"); - expect(interactive.strategy).toBe("interactive"); - expect(explicit.strategy).toBe("explicit"); - }); - it("RegistrySyncStatus should support all status values", () => { - const statuses: RegistrySyncStatus["status"][] = ["syncing", "success", "error", "offline"]; - expect(statuses).toHaveLength(4); + const statuses: RegistrySyncStatus["status"][] = ["success", "error", "offline"]; + expect(statuses).toHaveLength(3); }); }); diff --git a/tests/unit/reindex.test.ts b/tests/unit/reindex.test.ts index f6b5646e..8fb33f04 100644 --- a/tests/unit/reindex.test.ts +++ b/tests/unit/reindex.test.ts @@ -228,3 +228,36 @@ describe("reindex", () => { expect(result.completed).toBe(0); }); }); + +describe("reindex cancellation", () => { + const chunks = [ + { id: "c1", content: "chunk 1" }, + { id: "c2", content: "chunk 2" }, + { id: "c3", content: "chunk 3" }, + ]; + + it("rejects with the abort reason and stops between batches", async () => { + const { db } = createMockDb(chunks); + const { provider, embedBatchFn } = createMockProvider(); + const controller = new AbortController(); + + const promise = reindex(db, provider, { + batchSize: 1, + signal: controller.signal, + onProgress: () => controller.abort(), + }); + + await expect(promise).rejects.toHaveProperty("name", "AbortError"); + expect(embedBatchFn).toHaveBeenCalledTimes(1); + }); + + it("does not embed anything when the signal is already aborted", async () => { + const { db } = createMockDb(chunks); + const { provider, embedBatchFn } = createMockProvider(); + const controller = new AbortController(); + controller.abort(); + + await expect(reindex(db, provider, { signal: controller.signal })).rejects.toThrow(); + expect(embedBatchFn).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/repl.test.ts b/tests/unit/repl.test.ts index beb4ea92..fa1eace7 100644 --- a/tests/unit/repl.test.ts +++ b/tests/unit/repl.test.ts @@ -1,129 +1,60 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import type { MockInstance } from "vitest"; -import { startRepl } from "../../src/cli/repl.js"; -import type Database from "better-sqlite3"; -import type { EmbeddingProvider } from "../../src/providers/embedding.js"; -import { EventEmitter } from "node:events"; +import { startInteractiveSearch } from "../../src/cli/repl.js"; +import type { Interface } from "node:readline/promises"; -/** Minimal mock readline.Interface that yields pre-programmed answers. */ -function createMockInterface(inputs: string[]) { +/** Minimal readline.Interface that yields pre-programmed answers, then "closes". */ +function createMockInterface(inputs: string[]): { iface: Interface; closeFn: MockInstance } { const queue = [...inputs]; - const emitter = new EventEmitter(); - - const questionFn = vi.fn(() => { - const next = queue.shift(); - if (next === undefined) return Promise.reject(new Error("closed")); - return Promise.resolve(next); - }); const closeFn = vi.fn(); - const iface = { - question: questionFn, + question: vi.fn(() => { + const next = queue.shift(); + return next === undefined ? Promise.reject(new Error("closed")) : Promise.resolve(next); + }), close: closeFn, - on: emitter.on.bind(emitter), - } as unknown as import("node:readline/promises").Interface; - - return { iface, questionFn, closeFn }; -} - -function createMockDb() { - return {} as unknown as Database.Database; -} - -function createMockProvider() { - return { - dimensions: 3, - generateEmbedding: vi.fn(() => Promise.resolve(new Float32Array([0.1, 0.2, 0.3]))), - } as unknown as EmbeddingProvider; + } as unknown as Interface; + return { iface, closeFn }; } -describe("startRepl", () => { - let consoleSpy: MockInstance; - let consoleErrorSpy: MockInstance; +describe("startInteractiveSearch", () => { + let log: MockInstance; + let error: MockInstance; beforeEach(() => { - consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + log = vi.spyOn(console, "log").mockImplementation(() => {}); + error = vi.spyOn(console, "error").mockImplementation(() => {}); }); afterEach(() => { - consoleSpy.mockRestore(); - consoleErrorSpy.mockRestore(); - }); - - it("exits on 'quit' command", async () => { - const { iface, closeFn } = createMockInterface(["quit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(closeFn).toHaveBeenCalled(); - expect(consoleSpy).toHaveBeenCalledWith("Goodbye!"); - }); - - it("exits on 'exit' command", async () => { - const { iface, closeFn } = createMockInterface(["exit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(closeFn).toHaveBeenCalled(); - expect(consoleSpy).toHaveBeenCalledWith("Goodbye!"); - }); - - it("skips empty input lines", async () => { - const { iface, questionFn, closeFn } = createMockInterface(["", " ", "quit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(questionFn).toHaveBeenCalledTimes(3); - expect(closeFn).toHaveBeenCalled(); - }); - - it("exits gracefully on readline close (Ctrl+C)", async () => { - const { iface, closeFn } = createMockInterface([]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(closeFn).toHaveBeenCalled(); - }); - - it("handles search errors gracefully", async () => { - const { iface, closeFn } = createMockInterface(["test query", "quit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("Search error:")); - expect(closeFn).toHaveBeenCalled(); - }); - - it("prints banner on start", async () => { - const { iface } = createMockInterface(["quit"]); - - await startRepl({ - db: createMockDb(), - provider: createMockProvider(), - createInterface: () => iface, - }); - - expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining("LibScope interactive search")); + log.mockRestore(); + error.mockRestore(); + }); + + it("runs each non-empty query and stops on 'quit'", async () => { + const { iface, closeFn } = createMockInterface([" first ", "", "second", "quit", "never"]); + const onQuery = vi.fn(() => Promise.resolve()); + await startInteractiveSearch({ onQuery, createInterface: () => iface }); + expect(onQuery.mock.calls).toEqual([["first"], ["second"]]); + expect(closeFn).toHaveBeenCalledOnce(); + }); + + it("stops when input closes (Ctrl+D)", async () => { + const { iface, closeFn } = createMockInterface(["one"]); + const onQuery = vi.fn(() => Promise.resolve()); + await startInteractiveSearch({ onQuery, createInterface: () => iface }); + expect(onQuery).toHaveBeenCalledTimes(1); + expect(closeFn).toHaveBeenCalledOnce(); + }); + + it("prints an error and keeps going when a query fails", async () => { + const { iface } = createMockInterface(["bad", "good", "exit"]); + const onQuery = vi + .fn<(q: string) => Promise>() + .mockRejectedValueOnce(new Error("boom")) + .mockResolvedValueOnce(undefined); + await startInteractiveSearch({ onQuery, createInterface: () => iface }); + expect(error).toHaveBeenCalledWith("✗ boom"); + expect(onQuery).toHaveBeenCalledTimes(2); }); }); diff --git a/tests/unit/reporter.test.ts b/tests/unit/reporter.test.ts index b63d8d79..7fc4995e 100644 --- a/tests/unit/reporter.test.ts +++ b/tests/unit/reporter.test.ts @@ -1,172 +1,49 @@ -import { describe, it, expect, vi, afterEach } from "vitest"; -import { isVerbose, createReporter } from "../../src/cli/reporter.js"; - -describe("reporter", () => { - afterEach(() => { - delete process.env["LIBSCOPE_VERBOSE"]; - vi.restoreAllMocks(); +import { describe, it, expect, vi } from "vitest"; +import { createProgressLine, formatProgress } from "../../src/cli/reporter.js"; + +function fakeStream(isTTY: boolean): { stream: NodeJS.WriteStream; written: string[] } { + const written: string[] = []; + const stream = { + isTTY, + columns: 60, + write: vi.fn((s: string) => { + written.push(s); + return true; + }), + } as unknown as NodeJS.WriteStream; + return { stream, written }; +} + +describe("formatProgress", () => { + it("shows a bar, percent and counts when the total is known", () => { + const line = formatProgress({ done: 5, total: 10, message: "docs/a.md" }); + expect(line).toContain("50% (5/10) docs/a.md"); + expect(line).toContain("█".repeat(10) + "░".repeat(10)); }); - describe("isVerbose", () => { - it("returns true when verbose flag is set", () => { - expect(isVerbose(true)).toBe(true); - }); - - it("returns false when verbose flag is false", () => { - expect(isVerbose(false)).toBe(false); - }); - - it("returns false when verbose flag is undefined", () => { - expect(isVerbose(undefined)).toBe(false); - }); - - it("returns true when LIBSCOPE_VERBOSE=1 env var is set", () => { - process.env["LIBSCOPE_VERBOSE"] = "1"; - expect(isVerbose(false)).toBe(true); - }); - - it("returns false when LIBSCOPE_VERBOSE=0", () => { - process.env["LIBSCOPE_VERBOSE"] = "0"; - expect(isVerbose(false)).toBe(false); - }); + it("shows only the count without a total, and shortens long labels", () => { + const line = formatProgress({ done: 3, message: "x".repeat(80) }); + expect(line.startsWith("(3) ...")).toBe(true); + expect(line.length).toBeLessThan(60); }); +}); - describe("createReporter", () => { - it("returns a SilentReporter (no-op) in verbose mode", () => { - const reporter = createReporter(true); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); - - reporter.log("hello"); - reporter.success("done"); - reporter.warn("careful"); - reporter.error("bad"); - reporter.progress(1, 10, "task"); - reporter.clearProgress(); - - expect(stdout).not.toHaveBeenCalled(); - expect(stderr).not.toHaveBeenCalled(); - }); - - it("returns a SilentReporter when LIBSCOPE_VERBOSE=1", () => { - process.env["LIBSCOPE_VERBOSE"] = "1"; - const reporter = createReporter(); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.log("hello"); - expect(stdout).not.toHaveBeenCalled(); - }); - - it("PrettyReporter.log writes to stdout", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.log("test message"); - - expect(stdout).toHaveBeenCalledOnce(); - expect(String(stdout.mock.calls[0]![0])).toContain("test message"); - }); - - it("PrettyReporter.success writes green checkmark to stdout", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.success("all done"); - - const output = String(stdout.mock.calls[0]![0]); - expect(output).toContain("all done"); - // Green ANSI code - expect(output).toContain("\x1b[32m"); - }); - - it("PrettyReporter.warn writes to stderr", () => { - const reporter = createReporter(false); - const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); - - reporter.warn("watch out"); - - expect(stderr).toHaveBeenCalledOnce(); - expect(String(stderr.mock.calls[0]![0])).toContain("watch out"); - }); - - it("PrettyReporter.error writes to stderr", () => { - const reporter = createReporter(false); - const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); - - reporter.error("something failed"); - - expect(stderr).toHaveBeenCalledOnce(); - expect(String(stderr.mock.calls[0]![0])).toContain("something failed"); - }); - - it("PrettyReporter.progress writes \\r-prefixed line to stdout", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(3, 10, "indexing doc"); - - const output = String(stdout.mock.calls[0]![0]); - expect(output).toMatch(/^\r/); - expect(output).toContain("3/10"); - expect(output).toContain("30%"); - }); - - it("PrettyReporter.clearProgress clears the progress line", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(1, 5, "working"); - stdout.mockClear(); - - reporter.clearProgress(); - - // Should write spaces to clear the line - const output = String(stdout.mock.calls[0]![0]); - expect(output).toMatch(/^\r\s+\r$/); - }); - - it("PrettyReporter.clearProgress is a no-op when no progress shown", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.clearProgress(); - - expect(stdout).not.toHaveBeenCalled(); - }); - - it("PrettyReporter.log clears progress before writing", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(1, 5, "working"); - stdout.mockClear(); - - reporter.log("a message"); - - // First call should be the clear, second the message - expect(stdout.mock.calls.length).toBeGreaterThanOrEqual(2); - const clearCall = String(stdout.mock.calls[0]![0]); - expect(clearCall).toMatch(/^\r\s+\r$/); - }); - - it("PrettyReporter.progress truncates long labels", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(1, 1, "a".repeat(50)); - - const output = String(stdout.mock.calls[0]![0]); - expect(output).toContain("..."); - }); - - it("PrettyReporter.progress handles zero total gracefully", () => { - const reporter = createReporter(false); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - reporter.progress(0, 0, "starting"); +describe("createProgressLine", () => { + it("writes and clears one line on a terminal", () => { + const { stream, written } = fakeStream(true); + const progress = createProgressLine(stream); + progress.update({ done: 1, total: 2 }); + progress.clear(); + expect(written).toHaveLength(2); + expect(written[0]?.startsWith("\r[")).toBe(true); + expect(written[1]?.trim()).toBe(""); + }); - const output = String(stdout.mock.calls[0]![0]); - expect(output).toContain("0%"); - }); + it("writes nothing when the stream is not a terminal", () => { + const { stream, written } = fakeStream(false); + const progress = createProgressLine(stream); + progress.update({ done: 1, total: 2 }); + progress.clear(); + expect(written).toEqual([]); }); }); diff --git a/tests/unit/root-exports.test.ts b/tests/unit/root-exports.test.ts new file mode 100644 index 00000000..8730a4e1 --- /dev/null +++ b/tests/unit/root-exports.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; + +/** + * The runtime exports of the package entry points. Adding a public export is an API + * decision: update this list deliberately (and docs/reference/lite-api.md or + * docs/guide/programmatic-usage.md). + */ +const ROOT_EXPORTS = [ + "ChunkNotFoundError", + "ConfigError", + "DatabaseError", + "DocumentNotFoundError", + "EmbeddingError", + "FetchError", + "LibScope", + "LibScopeError", + "NotFoundError", + "TopicNotFoundError", + "ValidationError", +]; + +const LITE_EXTRAS = ["TreeSitterChunker", "createCodeChunker", "createLite", "normalizeRawInput"]; + +describe("package exports", () => { + it("libscope exports only LibScope and the error classes", async () => { + const root = await import("../../src/core/index.js"); + expect(Object.keys(root).sort()).toEqual(ROOT_EXPORTS); + }); + + it("libscope/lite exports the root plus the lite helpers", async () => { + const lite = await import("../../src/lite/index.js"); + expect(Object.keys(lite).sort()).toEqual([...ROOT_EXPORTS, ...LITE_EXTRAS].sort()); + }); +}); diff --git a/tests/unit/saved-searches.test.ts b/tests/unit/saved-searches.test.ts index b2eb001f..755c194e 100644 --- a/tests/unit/saved-searches.test.ts +++ b/tests/unit/saved-searches.test.ts @@ -1,15 +1,13 @@ import { describe, it, expect, beforeEach } from "vitest"; -import { createTestDb, createTestDbWithVec } from "../fixtures/test-db.js"; -import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { createTestDb } from "../fixtures/test-db.js"; import { createSavedSearch, listSavedSearches, getSavedSearch, deleteSavedSearch, - runSavedSearch, + recordSavedSearchRun, } from "../../src/core/saved-searches.js"; -import { indexDocument } from "../../src/core/indexing.js"; -import { ValidationError, DocumentNotFoundError } from "../../src/errors.js"; +import { ValidationError, NotFoundError } from "../../src/errors.js"; import type Database from "better-sqlite3"; import { initLogger } from "../../src/logger.js"; @@ -17,11 +15,9 @@ initLogger("silent"); describe("saved-searches", () => { let db: Database.Database; - let provider: MockEmbeddingProvider; beforeEach(() => { db = createTestDb(); - provider = new MockEmbeddingProvider(); }); describe("createSavedSearch", () => { @@ -90,7 +86,7 @@ describe("saved-searches", () => { }); it("should throw for nonexistent search", () => { - expect(() => getSavedSearch(db, "nonexistent")).toThrow(DocumentNotFoundError); + expect(() => getSavedSearch(db, "nonexistent")).toThrow(NotFoundError); }); }); @@ -108,69 +104,17 @@ describe("saved-searches", () => { }); it("should throw for nonexistent search", () => { - expect(() => deleteSavedSearch(db, "nonexistent")).toThrow(DocumentNotFoundError); + expect(() => deleteSavedSearch(db, "nonexistent")).toThrow(NotFoundError); }); }); - describe("runSavedSearch", () => { - it("should run a saved search and update metadata", async () => { - const vecDb = createTestDbWithVec(); - const vecProvider = new MockEmbeddingProvider(); - - await indexDocument(vecDb, vecProvider, { - title: "TypeScript Guide", - content: "# TypeScript\n\nTypeScript is a typed superset of JavaScript.", - sourceType: "manual", - }); - - const saved = createSavedSearch(vecDb, "TS Search", "typescript"); + describe("recordSavedSearchRun", () => { + it("records the run time and result count", () => { + const saved = createSavedSearch(db, "TS Search", "typescript"); expect(saved.lastRunAt).toBeNull(); - expect(saved.resultCount).toBe(0); - - const { search, results } = await runSavedSearch(vecDb, vecProvider, saved.id); - expect(search.lastRunAt).toBeTruthy(); - expect(search.resultCount).toBe(results.length); - expect(results.length).toBeGreaterThan(0); - }); - - it("should run by name", async () => { - const vecDb = createTestDbWithVec(); - const vecProvider = new MockEmbeddingProvider(); - - await indexDocument(vecDb, vecProvider, { - title: "Test Doc", - content: "# Test\n\nSome test content here.", - sourceType: "manual", - }); - - createSavedSearch(vecDb, "Run By Name", "test"); - const { search } = await runSavedSearch(vecDb, vecProvider, "Run By Name"); - expect(search.name).toBe("Run By Name"); - expect(search.lastRunAt).toBeTruthy(); - }); - - it("should apply saved filters", async () => { - const vecDb = createTestDbWithVec(); - const vecProvider = new MockEmbeddingProvider(); - - await indexDocument(vecDb, vecProvider, { - title: "Express Auth", - content: "# Auth\n\nAuthentication with Express.", - sourceType: "library", - library: "express", - }); - - const saved = createSavedSearch(vecDb, "Express Only", "auth", { library: "express" }); - const { results } = await runSavedSearch(vecDb, vecProvider, saved.id); - for (const r of results) { - expect(r.library).toBe("express"); - } - }); - - it("should throw for nonexistent search", async () => { - await expect(runSavedSearch(db, provider, "nonexistent")).rejects.toThrow( - DocumentNotFoundError, - ); + const updated = recordSavedSearchRun(db, saved.id, 3); + expect(updated.lastRunAt).toBeTruthy(); + expect(updated.resultCount).toBe(3); }); }); diff --git a/tests/unit/scheduler.test.ts b/tests/unit/scheduler.test.ts index d5a9c435..47c4310d 100644 --- a/tests/unit/scheduler.test.ts +++ b/tests/unit/scheduler.test.ts @@ -29,6 +29,7 @@ vi.mock("../../src/connectors/index.js", async (importOriginal) => { ...orig, loadNamedConnectorConfig: vi.fn().mockReturnValue({ token: "test-token" }), saveNamedConnectorConfig: vi.fn(), + hasNamedConnectorConfig: vi.fn().mockReturnValue(false), startSync: vi.fn().mockReturnValue("sync-id-1"), completeSync: vi.fn(), failSync: vi.fn(), @@ -57,7 +58,8 @@ vi.mock("../../src/connectors/onenote.js", () => ({ // Import after mocks are set up const { ConnectorScheduler, loadScheduleEntries } = await import("../../src/core/scheduler.js"); -const { startSync, completeSync, failSync } = await import("../../src/connectors/index.js"); +const { startSync, completeSync, failSync, saveNamedConnectorConfig } = + await import("../../src/connectors/index.js"); const { syncNotion } = await import("../../src/connectors/notion.js"); const { syncSlack } = await import("../../src/connectors/slack.js"); const { syncConfluence } = await import("../../src/connectors/confluence.js"); @@ -149,132 +151,46 @@ describe("ConnectorScheduler", () => { }); describe("runSync via cron callback", () => { - it("runs a notion sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { connectorType: "notion", connectorName: "my-notion", cronExpression: "0 */6 * * *" }, - ]); - - const callback = cronCallbacks.get("0 */6 * * *"); - expect(callback).toBeDefined(); - callback!(); - // Allow async runSync to complete - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(startSync).toHaveBeenCalledWith(db, "notion", "my-notion"); - expect(syncNotion).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 5, - updated: 0, - deleted: 0, - errored: 0, - }); - - const status = scheduler.getStatus(); - expect(status.jobs[0]!.lastRun).toBeDefined(); - expect(status.jobs[0]!.running).toBe(false); - await scheduler.stop(); - }); - - it("runs a slack sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { connectorType: "slack", connectorName: "my-slack", cronExpression: "0 0 * * *" }, - ]); - - cronCallbacks.get("0 0 * * *")!(); - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(syncSlack).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 12, - updated: 0, - deleted: 0, - errored: 0, - }); - await scheduler.stop(); - }); - - it("runs a confluence sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { - connectorType: "confluence", - connectorName: "my-confluence", - cronExpression: "0 */6 * * *", - }, - ]); - - cronCallbacks.get("0 */6 * * *")!(); - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(syncConfluence).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 3, - updated: 1, - deleted: 0, - errored: 0, - }); - await scheduler.stop(); - }); - - it("runs an obsidian sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { - connectorType: "obsidian", - connectorName: "my-obsidian", - cronExpression: "0 */6 * * *", - }, - ]); - - cronCallbacks.get("0 */6 * * *")!(); - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(syncObsidianVault).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 4, - updated: 2, - deleted: 1, - errored: 0, - }); - await scheduler.stop(); - }); - - it("runs a onenote sync successfully", async () => { - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { - connectorType: "onenote", - connectorName: "my-onenote", - cronExpression: "0 */6 * * *", - }, - ]); - - cronCallbacks.get("0 */6 * * *")!(); - await vi.waitFor(() => { - expect(completeSync).toHaveBeenCalled(); - }); - - expect(syncOneNote).toHaveBeenCalled(); - expect(completeSync).toHaveBeenCalledWith(db, "sync-id-1", { - added: 6, - updated: 0, - deleted: 0, - errored: 0, - }); - await scheduler.stop(); - }); + const cases = [ + ["notion", syncNotion], + ["slack", syncSlack], + ["confluence", syncConfluence], + ["obsidian", syncObsidianVault], + ["onenote", syncOneNote], + ] as const; + + it.each(cases)( + "delegates a %s run to the connector under the configured name", + async (type, syncFn) => { + const scheduler = new ConnectorScheduler(db, provider); + scheduler.start([ + { connectorType: type, connectorName: `my-${type}`, cronExpression: "0 */6 * * *" }, + ]); + + cronCallbacks.get("0 */6 * * *")!(); + await vi.waitFor(() => { + expect(scheduler.getStatus().jobs[0]!.lastRun).toBeDefined(); + }); + + expect(syncFn).toHaveBeenCalledWith( + db, + provider, + expect.objectContaining({ token: "test-token" }), + { syncName: `my-${type}` }, + ); + // The connector records the run; the scheduler must not add a second row. + expect(startSync).not.toHaveBeenCalled(); + expect(completeSync).not.toHaveBeenCalled(); + const [savedName, savedConfig] = vi.mocked(saveNamedConnectorConfig).mock.calls[0]!; + expect(savedName).toBe(`my-${type}`); + expect(savedConfig).toMatchObject({ connectorType: type }); + expect(typeof (savedConfig as { lastSync?: unknown }).lastSync).toBe("string"); + expect(scheduler.getStatus().jobs[0]!.running).toBe(false); + await scheduler.stop(); + }, + ); - it("handles sync failure and calls failSync", async () => { + it("does not save lastSync or write a sync row when the connector fails", async () => { vi.mocked(syncNotion).mockRejectedValueOnce(new Error("Network error")); const scheduler = new ConnectorScheduler(db, provider); @@ -284,17 +200,17 @@ describe("ConnectorScheduler", () => { cronCallbacks.get("*/5 * * * *")!(); await vi.waitFor(() => { - expect(failSync).toHaveBeenCalled(); + expect(scheduler.getStatus().jobs[0]!.lastRun).toBeDefined(); }); - expect(failSync).toHaveBeenCalledWith(db, "sync-id-1", "Network error"); - const status = scheduler.getStatus(); - expect(status.jobs[0]!.running).toBe(false); - expect(status.jobs[0]!.lastRun).toBeDefined(); + expect(saveNamedConnectorConfig).not.toHaveBeenCalled(); + expect(startSync).not.toHaveBeenCalled(); + expect(failSync).not.toHaveBeenCalled(); + expect(scheduler.getStatus().jobs[0]!.running).toBe(false); await scheduler.stop(); }); - it("handles unknown connector type", async () => { + it("records one failed row for an unknown connector type", async () => { const scheduler = new ConnectorScheduler(db, provider); scheduler.start([ { @@ -306,14 +222,22 @@ describe("ConnectorScheduler", () => { cronCallbacks.get("0 */6 * * *")!(); await vi.waitFor(() => { - expect(failSync).toHaveBeenCalled(); + expect(scheduler.getStatus().jobs[0]!.lastRun).toBeDefined(); }); - expect(failSync).toHaveBeenCalledWith( - db, - "sync-id-1", - "Unknown connector type: unknown-type", - ); + const rows = db.prepare("SELECT * FROM connector_syncs").all() as Array<{ + connector_type: string; + connector_name: string; + status: string; + error_message: string; + }>; + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + connector_type: "unknown-type", + connector_name: "my-unknown", + status: "failed", + error_message: "Unknown connector type: unknown-type", + }); await scheduler.stop(); }); @@ -323,7 +247,7 @@ describe("ConnectorScheduler", () => { vi.mocked(syncNotion).mockImplementationOnce( () => new Promise((resolve) => { - resolveSync = () => resolve({ pagesIndexed: 1, errors: [] }); + resolveSync = () => resolve({ pagesIndexed: 1, databasesIndexed: 0, errors: [] }); }), ); @@ -354,23 +278,6 @@ describe("ConnectorScheduler", () => { expect(syncNotion).toHaveBeenCalledTimes(1); await scheduler.stop(); }); - - it("handles non-Error thrown values in sync", async () => { - vi.mocked(syncNotion).mockRejectedValueOnce("string error"); - - const scheduler = new ConnectorScheduler(db, provider); - scheduler.start([ - { connectorType: "notion", connectorName: "str-err", cronExpression: "0 */6 * * *" }, - ]); - - cronCallbacks.get("0 */6 * * *")!(); - await vi.waitFor(() => { - expect(failSync).toHaveBeenCalled(); - }); - - expect(failSync).toHaveBeenCalledWith(db, "sync-id-1", "string error"); - await scheduler.stop(); - }); }); }); diff --git a/tests/unit/schema-migrations.test.ts b/tests/unit/schema-migrations.test.ts new file mode 100644 index 00000000..dec1f113 --- /dev/null +++ b/tests/unit/schema-migrations.test.ts @@ -0,0 +1,237 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import DatabaseConstructor from "better-sqlite3"; +import type Database from "better-sqlite3"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runMigrations } from "../../src/db/schema.js"; +import { searchDocuments } from "../../src/core/search.js"; +import { initLogger } from "../../src/logger.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; + +function newDb(): Database.Database { + const db = new DatabaseConstructor(":memory:"); + db.pragma("foreign_keys = ON"); + return db; +} + +function seedChunks(db: Database.Database, docCount: number, chunksPerDoc: number): void { + const insertDoc = db.prepare( + "INSERT INTO documents (id, source_type, title, content) VALUES (?, 'manual', ?, ?)", + ); + const insertChunk = db.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", + ); + for (let d = 0; d < docCount; d++) { + insertDoc.run(`doc-${d}`, `Document ${d}`, `content of document ${d}`); + for (let c = 0; c < chunksPerDoc; c++) { + insertChunk.run(`chunk-${d}-${c}`, `doc-${d}`, `alpha bravo doc${d} part${c}`, c); + } + } +} + +function seedChunksForDoc(db: Database.Database, docId: string, chunkCount: number): void { + db.prepare( + "INSERT INTO documents (id, source_type, title, content) VALUES (?, 'manual', ?, '')", + ).run(docId, docId); + const insertChunk = db.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, ?, ?)", + ); + for (let c = 0; c < chunkCount; c++) { + insertChunk.run(`${docId}-chunk-${c}`, docId, `charlie ${docId} part${c}`, c); + } +} + +function count(db: Database.Database, table: string): number { + return (db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n; +} + +/** FTS rows whose rowid does not point at the chunk they describe. */ +function misalignedFtsRows(db: Database.Database): number { + return ( + db + .prepare( + `SELECT COUNT(*) AS n FROM chunks_fts f + LEFT JOIN chunks c ON c.rowid = f.rowid + WHERE c.id IS NULL OR c.id != f.chunk_id`, + ) + .get() as { n: number } + ).n; +} + +async function keywordHits(db: Database.Database, query: string): Promise { + const { results } = await searchDocuments(db, new MockEmbeddingProvider(), { + query, + limit: 50, + }); + return results.map((r) => `${r.chunkId}:${r.score.toFixed(6)}`); +} + +describe("schema migrations", () => { + beforeEach(() => { + initLogger("silent"); + }); + + describe("FTS backfill", () => { + it("does not duplicate FTS rows when upgrading a populated database", () => { + const db = newDb(); + runMigrations(db, 16); + seedChunks(db, 3, 4); + expect(count(db, "chunks_fts")).toBe(12); + + runMigrations(db); + + expect(count(db, "chunks_fts")).toBe(count(db, "chunks")); + }); + + it("backfills an empty FTS index from existing chunks", () => { + const db = newDb(); + runMigrations(db, 16); + seedChunks(db, 2, 3); + db.exec("DELETE FROM chunks_fts"); + expect(count(db, "chunks_fts")).toBe(0); + + runMigrations(db); + + expect(count(db, "chunks_fts")).toBe(6); + }); + }); + + describe("migration 18: chunks_fts keyed by chunk rowid", () => { + it("keeps keyword search results unchanged", async () => { + const db = newDb(); + runMigrations(db, 17); + seedChunks(db, 4, 5); + const before = await keywordHits(db, "bravo doc2"); + expect(before.length).toBeGreaterThan(0); + + runMigrations(db); + + expect(await keywordHits(db, "bravo doc2")).toEqual(before); + expect(count(db, "chunks_fts")).toBe(count(db, "chunks")); + expect(misalignedFtsRows(db)).toBe(0); + }); + + it("removes duplicate FTS rows left by earlier backfills", async () => { + const db = newDb(); + runMigrations(db, 17); + seedChunks(db, 3, 4); + const clean = await keywordHits(db, "alpha"); + // Simulate the old backfill bug: every chunk indexed twice. + db.exec( + "INSERT INTO chunks_fts(content, chunk_id, document_id) SELECT content, id, document_id FROM chunks", + ); + expect(count(db, "chunks_fts")).toBe(24); + + runMigrations(db); + + expect(count(db, "chunks_fts")).toBe(12); + expect(await keywordHits(db, "alpha")).toEqual(clean); + }); + + it("keeps chunks_fts in sync on insert, update and delete", () => { + const db = newDb(); + runMigrations(db); + seedChunks(db, 3, 4); + expect(count(db, "chunks_fts")).toBe(12); + + db.prepare("UPDATE chunks SET content = 'zulu replaced' WHERE id = 'chunk-0-0'").run(); + const updated = db + .prepare("SELECT chunk_id FROM chunks_fts WHERE chunks_fts MATCH 'zulu'") + .all() as Array<{ chunk_id: string }>; + expect(updated).toEqual([{ chunk_id: "chunk-0-0" }]); + + db.prepare("DELETE FROM documents WHERE id = 'doc-1'").run(); + expect(count(db, "chunks_fts")).toBe(8); + expect(misalignedFtsRows(db)).toBe(0); + }); + + it("stays aligned with chunks after VACUUM", () => { + const dir = mkdtempSync(join(tmpdir(), "libscope-fts-")); + try { + const db = new DatabaseConstructor(join(dir, "test.db")); + db.pragma("foreign_keys = ON"); + runMigrations(db); + seedChunks(db, 6, 5); + db.prepare("DELETE FROM documents WHERE id IN ('doc-1', 'doc-3')").run(); + db.exec("VACUUM"); + expect(misalignedFtsRows(db)).toBe(0); + + db.prepare("DELETE FROM documents WHERE id = 'doc-4'").run(); + seedChunksForDoc(db, "doc-new", 3); + expect(count(db, "chunks_fts")).toBe(count(db, "chunks")); + expect(misalignedFtsRows(db)).toBe(0); + db.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + }); + + describe("migration 18: documents title index", () => { + it("indexes documents.title and uses it for the dedup lookup", () => { + const db = newDb(); + runMigrations(db, 17); + seedChunks(db, 2, 1); + + runMigrations(db); + + const index = db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_documents_title'", + ) + .get(); + expect(index).toBeDefined(); + const plan = db + .prepare( + "EXPLAIN QUERY PLAN SELECT id FROM documents WHERE title = ? AND LENGTH(content) = ?", + ) + .all("Document 1", 22) as Array<{ detail: string }>; + expect(plan.map((p) => p.detail).join(" ")).toContain("idx_documents_title"); + }); + }); + + describe("migration 19: saved search filter names", () => { + it("renames source to sourceType and keeps other rows and keys", () => { + const db = newDb(); + runMigrations(db, 18); + const insert = db.prepare( + "INSERT INTO saved_searches (id, name, query, filters) VALUES (?, ?, 'q', ?)", + ); + insert.run("a", "a", JSON.stringify({ source: "library", topic: "t", limit: 5 })); + insert.run("b", "b", JSON.stringify({ library: "react" })); + insert.run("c", "c", null); + insert.run("d", "d", "{not json"); + + runMigrations(db); + + const filters = (id: string): string | null => + ( + db.prepare("SELECT filters FROM saved_searches WHERE id = ?").get(id) as { + filters: string | null; + } + ).filters; + expect(JSON.parse(filters("a") ?? "null")).toEqual({ + sourceType: "library", + topic: "t", + limit: 5, + }); + expect(JSON.parse(filters("b") ?? "null")).toEqual({ library: "react" }); + expect(filters("c")).toBeNull(); + expect(filters("d")).toBe("{not json"); + expect(count(db, "saved_searches")).toBe(4); + }); + }); + + describe("migration 18: index_meta table", () => { + it("creates an empty index_meta table on upgrade", () => { + const db = newDb(); + runMigrations(db, 17); + seedChunks(db, 1, 1); + + runMigrations(db); + + expect(count(db, "index_meta")).toBe(0); + }); + }); +}); diff --git a/tests/unit/schema.test.ts b/tests/unit/schema.test.ts index e1f71c18..3b1cd2e8 100644 --- a/tests/unit/schema.test.ts +++ b/tests/unit/schema.test.ts @@ -38,7 +38,7 @@ describe("database schema", () => { const version = db.prepare("SELECT MAX(version) as v FROM schema_version").get() as { v: number; }; - expect(version.v).toBe(17); + expect(version.v).toBe(19); }); it("should create expected indexes", () => { @@ -131,22 +131,14 @@ describe("database schema", () => { runMigrations(db); }); - it("should throw DatabaseError for zero dimensions", () => { - expect(() => createVectorTable(db, 0)).toThrow(DatabaseError); - expect(() => createVectorTable(db, 0)).toThrow("Invalid vector dimensions"); - }); - - it("should throw DatabaseError for negative dimensions", () => { - expect(() => createVectorTable(db, -1)).toThrow(DatabaseError); - }); - - it("should throw DatabaseError for dimensions > 10000", () => { - expect(() => createVectorTable(db, 10001)).toThrow(DatabaseError); - expect(() => createVectorTable(db, 10001)).toThrow("Invalid vector dimensions"); - }); - - it("should throw DatabaseError for non-integer dimensions", () => { - expect(() => createVectorTable(db, 3.5)).toThrow(DatabaseError); + it.each([ + { label: "zero", dimensions: 0 }, + { label: "negative", dimensions: -1 }, + { label: "> 10000", dimensions: 10001 }, + { label: "non-integer", dimensions: 3.5 }, + ])("should throw DatabaseError for $label dimensions", ({ dimensions }) => { + expect(() => createVectorTable(db, dimensions)).toThrow(DatabaseError); + expect(() => createVectorTable(db, dimensions)).toThrow("Invalid vector dimensions"); }); it("should not throw for valid dimensions (vec0 may not be available)", () => { @@ -154,88 +146,4 @@ describe("database schema", () => { expect(() => createVectorTable(db, 384)).not.toThrow(); }); }); - - describe("createVectorTable", () => { - it("should throw on invalid dimensions (zero)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 0)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (negative)", () => { - runMigrations(db); - expect(() => createVectorTable(db, -1)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (too large)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 10001)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on non-integer dimensions", () => { - runMigrations(db); - expect(() => createVectorTable(db, 3.5)).toThrow("Invalid vector dimensions"); - }); - - it("should not throw on valid dimensions (vec0 will fail without sqlite-vec but catches)", () => { - runMigrations(db); - // This won't create a real vector table (no sqlite-vec) but should not throw - createVectorTable(db, 384); - }); - }); - - describe("createVectorTable", () => { - it("should throw on invalid dimensions (zero)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 0)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (negative)", () => { - runMigrations(db); - expect(() => createVectorTable(db, -1)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (too large)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 10001)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on non-integer dimensions", () => { - runMigrations(db); - expect(() => createVectorTable(db, 3.5)).toThrow("Invalid vector dimensions"); - }); - - it("should not throw on valid dimensions (vec0 will fail without sqlite-vec but catches)", () => { - runMigrations(db); - // This won't create a real vector table (no sqlite-vec) but should not throw - createVectorTable(db, 384); - }); - }); - - describe("createVectorTable", () => { - it("should throw on invalid dimensions (zero)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 0)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (negative)", () => { - runMigrations(db); - expect(() => createVectorTable(db, -1)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on invalid dimensions (too large)", () => { - runMigrations(db); - expect(() => createVectorTable(db, 10001)).toThrow("Invalid vector dimensions"); - }); - - it("should throw on non-integer dimensions", () => { - runMigrations(db); - expect(() => createVectorTable(db, 3.5)).toThrow("Invalid vector dimensions"); - }); - - it("should not throw on valid dimensions (vec0 will fail without sqlite-vec but catches)", () => { - runMigrations(db); - // This won't create a real vector table (no sqlite-vec) but should not throw - createVectorTable(db, 384); - }); - }); }); diff --git a/tests/unit/search.test.ts b/tests/unit/search.test.ts index fa092412..eee9c619 100644 --- a/tests/unit/search.test.ts +++ b/tests/unit/search.test.ts @@ -28,7 +28,7 @@ describe("searchDocuments (FTS5 fallback)", () => { const { results, totalCount } = await searchDocuments(db, provider, { query: "TypeScript" }); - expect(results.length).toBe(2); + expect(results).toHaveLength(2); expect(totalCount).toBe(2); expect(results[0].content).toContain("TypeScript"); expect(results[0].score).toBeGreaterThan(0); @@ -46,7 +46,7 @@ describe("searchDocuments (FTS5 fallback)", () => { library: "react", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].library).toBe("react"); }); @@ -65,7 +65,7 @@ describe("searchDocuments (FTS5 fallback)", () => { topic: "testing", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].topicId).toBe("testing"); }); @@ -97,7 +97,7 @@ describe("searchDocuments (FTS5 fallback)", () => { }); expect(page1.totalCount).toBe(3); - expect(page1.results.length).toBe(2); + expect(page1.results).toHaveLength(2); const page2 = await searchDocuments(db, provider, { query: "TypeScript", @@ -106,7 +106,7 @@ describe("searchDocuments (FTS5 fallback)", () => { }); expect(page2.totalCount).toBe(3); - expect(page2.results.length).toBe(1); + expect(page2.results).toHaveLength(1); }); it("should filter by dateFrom", async () => { @@ -121,7 +121,7 @@ describe("searchDocuments (FTS5 fallback)", () => { dateFrom: "2024-01-01T00:00:00.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("New Doc"); }); @@ -137,7 +137,7 @@ describe("searchDocuments (FTS5 fallback)", () => { dateTo: "2023-12-31T23:59:59.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("Old Doc"); }); @@ -157,7 +157,7 @@ describe("searchDocuments (FTS5 fallback)", () => { dateTo: "2024-12-31T23:59:59.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("Mid Doc"); }); @@ -173,7 +173,7 @@ describe("searchDocuments (FTS5 fallback)", () => { source: "library", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].sourceType).toBe("library"); }); @@ -195,7 +195,7 @@ describe("searchDocuments (FTS5 fallback)", () => { version: "18.0.0", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].version).toBe("18.0.0"); }); @@ -221,7 +221,7 @@ describe("searchDocuments (FTS5 fallback)", () => { minRating: 4, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("Good Doc"); }); @@ -239,7 +239,7 @@ describe("searchDocuments (FTS5 fallback)", () => { tags: ["tutorial"], }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0].title).toBe("Tagged Doc"); }); @@ -252,7 +252,7 @@ describe("searchDocuments (FTS5 fallback)", () => { analyticsEnabled: false, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); // Verify no search log entry was created const logCount = db.prepare("SELECT COUNT(*) as cnt FROM search_log").get() as { cnt: number }; expect(logCount.cnt).toBe(0); @@ -337,7 +337,7 @@ describe("LIKE wildcard escaping in keyword search (issue #79)", () => { insertChunk(db, "c2", "doc2", "something completely different here"); const { results } = await searchDocuments(db, provider, { query: "100%" }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.content).toContain("100%"); }); @@ -349,7 +349,7 @@ describe("LIKE wildcard escaping in keyword search (issue #79)", () => { insertChunk(db, "c2", "doc2", "the username field is optional"); const { results } = await searchDocuments(db, provider, { query: "user_name" }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.content).toContain("user_name"); }); }); @@ -406,7 +406,7 @@ describe("search result scoring explanation (issue #89)", () => { const { results } = await searchDocuments(db, provider, { query: "consistency" }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); const result = results[0]!; // FTS5: score = -rawScore (BM25 rank is negative) expect(result.score).toBe(-result.scoreExplanation.rawScore); @@ -454,7 +454,7 @@ describe("LIKE fallback with filters", () => { version: "18.0.0", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.version).toBe("18.0.0"); }); @@ -482,7 +482,7 @@ describe("LIKE fallback with filters", () => { minRating: 4, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.title).toBe("Rated Doc"); }); @@ -500,7 +500,7 @@ describe("LIKE fallback with filters", () => { dateFrom: "2024-01-01T00:00:00.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.title).toBe("New Doc"); }); @@ -518,7 +518,7 @@ describe("LIKE fallback with filters", () => { dateTo: "2023-12-31T23:59:59.000Z", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.title).toBe("Old Doc"); }); @@ -536,7 +536,7 @@ describe("LIKE fallback with filters", () => { source: "library", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.sourceType).toBe("library"); }); @@ -556,7 +556,7 @@ describe("LIKE fallback with filters", () => { tags: ["howto"], }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.title).toBe("Tagged Doc"); }); @@ -583,7 +583,7 @@ describe("LIKE fallback with filters", () => { maxChunksPerDocument: 1, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); }); it("should attach context chunks in LIKE fallback", async () => { @@ -599,11 +599,11 @@ describe("LIKE fallback with filters", () => { contextChunks: 1, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.contextBefore).toBeDefined(); - expect(results[0]!.contextBefore!.length).toBe(1); + expect(results[0]!.contextBefore!).toHaveLength(1); expect(results[0]!.contextAfter).toBeDefined(); - expect(results[0]!.contextAfter!.length).toBe(1); + expect(results[0]!.contextAfter!).toHaveLength(1); }); }); @@ -628,7 +628,7 @@ describe("deduplicate search results by document (issue #245)", () => { const { results } = await searchDocuments(db, provider, { query: "TypeScript" }); - expect(results.length).toBe(3); + expect(results).toHaveLength(3); expect(results.every((r) => r.documentId === "doc1")).toBe(true); }); @@ -650,8 +650,8 @@ describe("deduplicate search results by document (issue #245)", () => { const doc1Results = results.filter((r) => r.documentId === "doc1"); const doc2Results = results.filter((r) => r.documentId === "doc2"); - expect(doc1Results.length).toBe(1); - expect(doc2Results.length).toBe(1); + expect(doc1Results).toHaveLength(1); + expect(doc2Results).toHaveLength(1); }); it("should limit to at most 2 chunks per document with maxChunksPerDocument=2", async () => { @@ -665,7 +665,7 @@ describe("deduplicate search results by document (issue #245)", () => { maxChunksPerDocument: 2, }); - expect(results.length).toBe(2); + expect(results).toHaveLength(2); expect(results.every((r) => r.documentId === "doc1")).toBe(true); }); }); @@ -695,14 +695,14 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 1, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); const result = results[0]!; expect(result.content).toContain("Core concepts"); expect(result.contextBefore).toBeDefined(); - expect(result.contextBefore!.length).toBe(1); + expect(result.contextBefore!).toHaveLength(1); expect(result.contextBefore![0]!.content).toContain("Introduction"); expect(result.contextAfter).toBeDefined(); - expect(result.contextAfter!.length).toBe(1); + expect(result.contextAfter!).toHaveLength(1); expect(result.contextAfter![0]!.content).toContain("Advanced patterns"); }); @@ -719,12 +719,12 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 2, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); const result = results[0]!; - expect(result.contextBefore!.length).toBe(2); + expect(result.contextBefore!).toHaveLength(2); expect(result.contextBefore![0]!.content).toContain("Section zero"); expect(result.contextBefore![1]!.content).toContain("Section one"); - expect(result.contextAfter!.length).toBe(2); + expect(result.contextAfter!).toHaveLength(2); expect(result.contextAfter![0]!.content).toContain("Section three"); expect(result.contextAfter![1]!.content).toContain("Section four"); }); @@ -739,9 +739,9 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 1, }); - expect(results.length).toBe(1); - expect(results[0]!.contextBefore!.length).toBe(0); - expect(results[0]!.contextAfter!.length).toBe(1); + expect(results).toHaveLength(1); + expect(results[0]!.contextBefore!).toHaveLength(0); + expect(results[0]!.contextAfter!).toHaveLength(1); }); it("should handle last chunk with no context after", async () => { @@ -754,9 +754,9 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 1, }); - expect(results.length).toBe(1); - expect(results[0]!.contextBefore!.length).toBe(1); - expect(results[0]!.contextAfter!.length).toBe(0); + expect(results).toHaveLength(1); + expect(results[0]!.contextBefore!).toHaveLength(1); + expect(results[0]!.contextAfter!).toHaveLength(0); }); it("should not include context when contextChunks is 0 or unset", async () => { @@ -769,7 +769,7 @@ describe("context chunk expansion (issue #247)", () => { query: "keyword", }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.contextBefore).toBeUndefined(); expect(results[0]!.contextAfter).toBeUndefined(); }); @@ -789,10 +789,10 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 5, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); // Capped at 2 - expect(results[0]!.contextBefore!.length).toBe(2); - expect(results[0]!.contextAfter!.length).toBe(2); + expect(results[0]!.contextBefore!).toHaveLength(2); + expect(results[0]!.contextAfter!).toHaveLength(2); }); it("should include chunkIndex in context chunks", async () => { @@ -806,7 +806,7 @@ describe("context chunk expansion (issue #247)", () => { contextChunks: 1, }); - expect(results.length).toBe(1); + expect(results).toHaveLength(1); expect(results[0]!.contextBefore![0]!.chunkIndex).toBe(0); expect(results[0]!.contextAfter![0]!.chunkIndex).toBe(2); }); @@ -854,6 +854,17 @@ describe("FTS5 AND-by-default logic (issue #362)", () => { expect(results.length).toBeGreaterThan(0); expect(results[0]!.content).toContain("TypeScript"); }); + + it("should leave function words out of the OR fallback", async () => { + insertDoc(db, "doc1", "Deploy Guide"); + insertChunk(db, "c1", "doc1", "Deploy to Kubernetes using Helm charts"); + insertDoc(db, "doc2", "Auth Guide"); + insertChunk(db, "c2", "doc2", "Authenticate every request with an OAuth2 token"); + + const { results } = await searchDocuments(db, provider, { query: "how to authenticate" }); + + expect(results.map((r) => r.documentId)).toEqual(["doc2"]); + }); }); describe("title boosting (issue #362)", () => { @@ -881,7 +892,7 @@ describe("title boosting (issue #362)", () => { }); // Both match the keyword, but doc1's title matches the query - expect(results.length).toBe(2); + expect(results).toHaveLength(2); const doc1Result = results.find((r) => r.documentId === "doc1"); const doc2Result = results.find((r) => r.documentId === "doc2"); expect(doc1Result).toBeDefined(); @@ -901,8 +912,8 @@ describe("title boosting (issue #362)", () => { query: "TypeScript", }); - expect(results.length).toBe(1); - expect(results[0]!.scoreExplanation.boostFactors.length).toBe(0); + expect(results).toHaveLength(1); + expect(results[0]!.scoreExplanation.boostFactors).toHaveLength(0); }); }); @@ -948,7 +959,7 @@ describe("lazy count optimization (issue #362)", () => { }); expect(page1.totalCount).toBeGreaterThanOrEqual(5); - expect(page1.results.length).toBe(2); + expect(page1.results).toHaveLength(2); }); }); diff --git a/tests/unit/slack.test.ts b/tests/unit/slack.test.ts index 79a53d5b..445c1c6a 100644 --- a/tests/unit/slack.test.ts +++ b/tests/unit/slack.test.ts @@ -4,6 +4,7 @@ import { syncSlack, disconnectSlack, _setRateLimitDelay, + toSlackTimestamp, _clearUserCache, } from "../../src/connectors/slack.js"; import type { SlackConfig } from "../../src/connectors/slack.js"; @@ -230,7 +231,7 @@ describe("syncSlack", () => { expect(result.messagesIndexed).toBe(2); }); - it("uses oldest param for incremental sync", async () => { + it("sends lastSync to Slack as Unix seconds in the oldest param", async () => { const syncTime = "2024-01-01T00:00:00.000Z"; setupChannelList([{ id: "C001", name: "general" }]); setupMessages([]); @@ -242,7 +243,14 @@ describe("syncSlack", () => { (call: unknown[]) => typeof call[0] === "string" && call[0].includes("conversations.history"), ); expect(historyCall).toBeDefined(); - expect(String(historyCall?.[0])).toContain("oldest=" + encodeURIComponent(syncTime)); + expect(new URL(String(historyCall?.[0])).searchParams.get("oldest")).toBe("1704067200"); + }); + + it("toSlackTimestamp passes numeric values through and drops invalid ones", () => { + expect(toSlackTimestamp("1704067200.5")).toBe("1704067200.5"); + expect(toSlackTimestamp("2024-01-01T00:00:01.500Z")).toBe("1704067201.5"); + expect(toSlackTimestamp("not a date")).toBeUndefined(); + expect(toSlackTimestamp(undefined)).toBeUndefined(); }); it("resolves user mentions in message text", async () => { diff --git a/tests/unit/spider.test.ts b/tests/unit/spider.test.ts index 24d93488..f64c5aaa 100644 --- a/tests/unit/spider.test.ts +++ b/tests/unit/spider.test.ts @@ -81,6 +81,18 @@ describe("spiderUrl", () => { expect(page.depth).toBe(0); }); + it("decodes HTML character references in the page title", async () => { + mockFetchRaw.mockImplementation((url: string) => { + if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); + return Promise.resolve(pageResponse(htmlPage("Docs — A & B"), url)); + }); + + const gen = spiderUrl("https://example.com/", { maxPages: 1, requestDelay: 0 }); + const result = await gen.next(); + const page = result.value as { title: string }; + expect(page.title).toBe("Docs — A & B"); + }); + it("follows links up to maxDepth", async () => { mockFetchRaw.mockImplementation((url: string) => { if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); @@ -165,8 +177,8 @@ describe("spiderUrl", () => { // Should only visit a and b once each const urls = pages.map((p) => p.url); - expect(urls.filter((u) => u === "https://example.com/a").length).toBe(1); - expect(urls.filter((u) => u === "https://example.com/b").length).toBe(1); + expect(urls.filter((u) => u === "https://example.com/a")).toHaveLength(1); + expect(urls.filter((u) => u === "https://example.com/b")).toHaveLength(1); }); it("filters cross-domain links when sameDomain=true (default)", async () => { @@ -425,7 +437,7 @@ describe("spiderUrl", () => { const gen = spiderUrl("https://example.com/", { maxDepth: 0, maxPages: 10, requestDelay: 0 }); const { pages } = await collectPages(gen); - expect(pages.length).toBe(1); + expect(pages).toHaveLength(1); expect(pages[0]!.url).toBe("https://example.com/"); }); @@ -476,7 +488,7 @@ describe("spiderUrl", () => { const gen = spiderUrl("https://example.com/notes.txt", { maxDepth: 0, requestDelay: 0 }); const { pages } = await collectPages(gen); - expect(pages.length).toBe(1); + expect(pages).toHaveLength(1); expect(pages[0]!.title).toBe("Plain Text"); }); @@ -494,4 +506,41 @@ describe("spiderUrl", () => { const { stats } = await collectPages(gen); expect((stats as { abortReason?: string }).abortReason).toBe("maxPages"); }); + + it("throws the abort reason between pages when the signal is aborted", async () => { + let counter = 0; + mockFetchRaw.mockImplementation((url: string) => { + if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); + counter++; + const links = [`https://example.com/page${counter + 100}`]; + return Promise.resolve(pageResponse(htmlPage(`Page ${counter}`, links), url)); + }); + const controller = new AbortController(); + + const gen = spiderUrl("https://example.com/", { + maxPages: 10, + maxDepth: 5, + requestDelay: 0, + signal: controller.signal, + }); + const first = await gen.next(); + expect(first.done).toBe(false); + controller.abort(); + + await expect(gen.next()).rejects.toBe(controller.signal.reason); + // Only the seed page was fetched (robots.txt calls excluded). + expect(counter).toBe(1); + }); + + it("throws the abort reason when a page fetch finishes after abort", async () => { + const controller = new AbortController(); + mockFetchRaw.mockImplementation((url: string) => { + if (url.endsWith("/robots.txt")) return Promise.reject(new Error("404")); + controller.abort(); + return Promise.resolve(pageResponse(htmlPage("Seed"), url)); + }); + + const gen = spiderUrl("https://example.com/", { requestDelay: 0, signal: controller.signal }); + await expect(gen.next()).rejects.toHaveProperty("name", "AbortError"); + }); }); diff --git a/tests/unit/tags.test.ts b/tests/unit/tags.test.ts index 2992de52..13e25120 100644 --- a/tests/unit/tags.test.ts +++ b/tests/unit/tags.test.ts @@ -5,7 +5,7 @@ import { deleteTag, listTags, addTagsToDocument, - removeTagFromDocument, + removeTagsFromDocument, getDocumentTags, getDocumentsByTag, suggestTags, @@ -110,13 +110,12 @@ describe("tags", () => { }); }); - describe("removeTagFromDocument", () => { - it("should remove only the specified tag", () => { + describe("removeTagsFromDocument", () => { + it("should remove only the named tags and report them", () => { insertDocument(db, "doc-1", "Test Doc"); - const tags = addTagsToDocument(db, "doc-1", ["keep", "remove"]); - const removeId = tags.find((t) => t.name === "remove")!.id; + addTagsToDocument(db, "doc-1", ["keep", "remove"]); - removeTagFromDocument(db, "doc-1", removeId); + expect(removeTagsFromDocument(db, "doc-1", ["REMOVE", "absent"])).toEqual(["remove"]); const docTags = getDocumentTags(db, "doc-1"); expect(docTags).toHaveLength(1); diff --git a/tests/unit/tasks.test.ts b/tests/unit/tasks.test.ts index 5aa32118..d200d4af 100644 --- a/tests/unit/tasks.test.ts +++ b/tests/unit/tasks.test.ts @@ -1,6 +1,11 @@ -import { describe, it, expect, beforeEach } from "vitest"; -import { TaskRegistry } from "../../src/mcp/tasks.js"; -import type { TaskType } from "../../src/mcp/tasks.js"; +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { TaskRegistry, isAbortRejection } from "../../src/core/tasks.js"; +import type { TaskType } from "../../src/core/tasks.js"; +import { reindex } from "../../src/core/reindex.js"; +import { initLogger } from "../../src/logger.js"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { insertChunk, insertDoc } from "../fixtures/helpers.js"; function makeRegistry(): TaskRegistry { return new TaskRegistry(); @@ -23,7 +28,7 @@ describe("TaskRegistry", () => { }); it("returns an AbortSignal that is not yet aborted", () => { - const { signal } = registry.create("reindex_library"); + const { signal } = registry.create("reindex_documents"); expect(signal.aborted).toBe(false); }); @@ -36,7 +41,7 @@ describe("TaskRegistry", () => { it("supports all task types", () => { const types: TaskType[] = [ "index_document", - "reindex_library", + "reindex_documents", "sync_connector", "install_pack", ]; @@ -60,7 +65,7 @@ describe("TaskRegistry", () => { }); it("prunes tasks whose completedAt is older than 1 hour", () => { - const { task } = registry.create("reindex_library"); + const { task } = registry.create("reindex_documents"); registry.update(task.id, { status: "completed", completedAt: new Date(Date.now() - 61 * 60 * 1000), // 61 minutes ago @@ -88,7 +93,7 @@ describe("TaskRegistry", () => { }); it("updates progress fields", () => { - const { task } = registry.create("reindex_library"); + const { task } = registry.create("reindex_documents"); registry.update(task.id, { progress: { current: 10, total: 100 } }); const updated = registry.get(task.id); expect(updated?.progress?.current).toBe(10); @@ -127,7 +132,7 @@ describe("TaskRegistry", () => { }); it("returns already_terminal for a failed task", () => { - const { task } = registry.create("reindex_library"); + const { task } = registry.create("reindex_documents"); registry.update(task.id, { status: "failed", completedAt: new Date() }); expect(registry.cancel(task.id)).toBe("already_terminal"); }); @@ -139,7 +144,7 @@ describe("TaskRegistry", () => { }); it("aborts the signal when cancelling a running task", () => { - const { task, signal } = registry.create("reindex_library"); + const { task, signal } = registry.create("reindex_documents"); registry.update(task.id, { status: "running", startedAt: new Date() }); const outcome = registry.cancel(task.id); expect(outcome).toBe("cancelled"); @@ -149,6 +154,106 @@ describe("TaskRegistry", () => { }); }); + describe("run", () => { + it("marks the task completed with the work's result", async () => { + const { task, done } = registry.run("index_document", (_signal, onProgress) => { + onProgress(1, 2); + return Promise.resolve("all done"); + }); + expect(registry.get(task.id)?.status).toBe("running"); + await done; + const finished = registry.get(task.id); + expect(finished?.status).toBe("completed"); + expect(finished?.result).toBe("all done"); + expect(finished?.progress).toEqual({ current: 1, total: 2 }); + expect(finished?.completedAt).toBeInstanceOf(Date); + }); + + it("marks the task failed when the work rejects without cancellation", async () => { + const { task, done } = registry.run("install_pack", () => Promise.reject(new Error("boom"))); + await done; + expect(registry.get(task.id)?.status).toBe("failed"); + expect(registry.get(task.id)?.error).toBe("boom"); + }); + + it("marks the task failed when the work throws synchronously", async () => { + const { task, done } = registry.run("install_pack", () => { + throw new Error("sync boom"); + }); + await done; + expect(registry.get(task.id)?.status).toBe("failed"); + expect(registry.get(task.id)?.error).toBe("sync boom"); + }); + + it("marks the task completed when the work finishes even though cancel was requested", async () => { + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + const { task, done } = registry.run("index_document", async () => { + await gate; + return "finished anyway"; + }); + expect(registry.cancel(task.id)).toBe("cancelled"); + release(); + await done; + expect(registry.get(task.id)?.status).toBe("completed"); + expect(registry.get(task.id)?.result).toBe("finished anyway"); + }); + + it("marks the task cancelled when the work rejects because of the abort", async () => { + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + const { task, done } = registry.run("reindex_documents", async (signal) => { + await gate; + signal.throwIfAborted(); + return "unreachable"; + }); + registry.cancel(task.id); + release(); + await done; + expect(registry.get(task.id)?.status).toBe("cancelled"); + expect(registry.get(task.id)?.error).toBeUndefined(); + }); + + it("marks the task failed when an unrelated error happens after cancel", async () => { + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + const { task, done } = registry.run("install_pack", async () => { + await gate; + throw new Error("disk full"); + }); + registry.cancel(task.id); + release(); + await done; + expect(registry.get(task.id)?.status).toBe("failed"); + expect(registry.get(task.id)?.error).toBe("disk full"); + }); + }); + + describe("isAbortRejection", () => { + it("is false when the signal is not aborted", () => { + const controller = new AbortController(); + const err = new Error("x"); + err.name = "AbortError"; + expect(isAbortRejection(err, controller.signal)).toBe(false); + }); + + it("matches the signal's reason and AbortError-named errors once aborted", () => { + const controller = new AbortController(); + controller.abort(); + expect(isAbortRejection(controller.signal.reason, controller.signal)).toBe(true); + const named = new Error("stopped"); + named.name = "AbortError"; + expect(isAbortRejection(named, controller.signal)).toBe(true); + expect(isAbortRejection(new Error("other"), controller.signal)).toBe(false); + }); + }); + describe("TTL pruning", () => { it("does not prune tasks without a completedAt", () => { const { task } = registry.create("index_document"); @@ -172,7 +277,7 @@ describe("TaskRegistry", () => { describe("async task lifecycle simulation", () => { it("transitions through pending -> running -> completed", () => { - const { task, signal } = registry.create("reindex_library"); + const { task, signal } = registry.create("reindex_documents"); expect(task.status).toBe("pending"); registry.update(task.id, { status: "running", startedAt: new Date() }); @@ -236,7 +341,39 @@ describe("TaskRegistry", () => { describe("taskRegistry singleton", () => { it("exports a module-level TaskRegistry instance", async () => { - const { taskRegistry } = await import("../../src/mcp/tasks.js"); + const { taskRegistry } = await import("../../src/core/tasks.js"); expect(taskRegistry).toBeInstanceOf(TaskRegistry); }); }); + +describe("TaskRegistry.run with core operations", () => { + it("reports cancelled when cancel-task aborts a running reindex", async () => { + initLogger("silent"); + const db = createTestDbWithVec(); + const provider = new MockEmbeddingProvider(); + insertDoc(db, "d1", "Doc"); + for (let i = 0; i < 3; i++) insertChunk(db, `c${i}`, "d1", `chunk ${i}`, i); + const embedSpy = vi.spyOn(provider, "embedBatch"); + + const registry = new TaskRegistry(); + let taskId = ""; + const { task, done } = registry.run("reindex_documents", async (signal, onProgress) => { + await reindex(db, provider, { + batchSize: 1, + signal, + onProgress: (p) => { + onProgress(p.completed, p.total); + registry.cancel(taskId); + }, + }); + return "Reindex complete."; + }); + taskId = task.id; + await done; + + expect(registry.get(task.id)?.status).toBe("cancelled"); + expect(registry.get(task.id)?.progress).toEqual({ current: 1, total: 3 }); + expect(embedSpy).toHaveBeenCalledTimes(1); + db.close(); + }); +}); diff --git a/tests/unit/text-utils.test.ts b/tests/unit/text-utils.test.ts new file mode 100644 index 00000000..91bf36bd --- /dev/null +++ b/tests/unit/text-utils.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { firstMarkdownHeading } from "../../src/utils/markdown.js"; +import { trimTrailingSlashes } from "../../src/utils/strings.js"; + +describe("firstMarkdownHeading", () => { + it.each([ + ["a heading on the first line", "# Title\ntext", "Title"], + ["a heading on a later line", "intro\n## not this\n# Real \nmore", "Real "], + ["a heading after \\r", "intro\r# Title", "Title"], + ["a heading after a Unicode line separator", "intro
# Title", "Title"], + ["the text after several spaces and tabs", "# \t Spaced", "Spaced"], + ["text on the next line when # is alone", "#\n\nNext line\nother", "Next line"], + ["a later heading when the first # is not a heading", "#tag\n# Heading", "Heading"], + ["the last space when only whitespace follows #", "# \n\n", " "], + ["the last tab before trailing line breaks", "# \t\n", "\t"], + ])("returns %s", (_case, text, expected) => { + expect(firstMarkdownHeading(text)).toBe(expected); + }); + + it.each([ + ["no heading", "plain text"], + ["# that is not at a line start", "text # not a heading"], + ["# followed by text without a space", "#hashtag"], + ["# followed only by line breaks", "#\n\n"], + ["# followed by one space only", "# "], + ["# followed by one space and line breaks", "# \n\n"], + ["an empty string", ""], + ])("returns undefined for %s", (_case, text) => { + expect(firstMarkdownHeading(text)).toBeUndefined(); + }); + + it("scans long whitespace runs quickly", () => { + const started = Date.now(); + expect(firstMarkdownHeading("#" + " \n".repeat(100_000))).toBe(" "); + expect(firstMarkdownHeading("#x\n".repeat(100_000))).toBeUndefined(); + expect(Date.now() - started).toBeLessThan(1000); + }); +}); + +describe("trimTrailingSlashes", () => { + it.each([ + ["https://x.org///", "https://x.org"], + ["https://x.org", "https://x.org"], + ["/", ""], + ["", ""], + ["a/b/", "a/b"], + ])("%j -> %j", (input, expected) => { + expect(trimTrailingSlashes(input)).toBe(expected); + }); +}); diff --git a/tests/unit/topics.test.ts b/tests/unit/topics.test.ts index 03035533..22717028 100644 --- a/tests/unit/topics.test.ts +++ b/tests/unit/topics.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach } from "vitest"; -import { createTestDb } from "../fixtures/test-db.js"; +import { createTestDb, createTestDbWithVec } from "../fixtures/test-db.js"; import { createTopic, listTopics, @@ -70,7 +70,7 @@ describe("topics", () => { createTopic(db, { name: "API" }); const topics = listTopics(db); - expect(topics.length).toBe(3); + expect(topics).toHaveLength(3); // Should be alphabetical expect(topics[0]!.name).toBe("API"); expect(topics[1]!.name).toBe("Auth"); @@ -84,12 +84,12 @@ describe("topics", () => { createTopic(db, { name: "Unrelated" }); const children = listTopics(db, parent.id); - expect(children.length).toBe(2); + expect(children).toHaveLength(2); }); it("should return empty array when no topics exist", () => { const topics = listTopics(db); - expect(topics.length).toBe(0); + expect(topics).toHaveLength(0); }); }); @@ -141,6 +141,43 @@ describe("topics", () => { expect(doc).toBeUndefined(); }); + it("should delete vectors of deleted documents when deleteDocuments is true", () => { + const vdb = createTestDbWithVec(); + createTopic(vdb, { name: "Vectors" }); + const insertDoc = vdb.prepare( + "INSERT INTO documents (id, source_type, title, content, topic_id) VALUES (?, 'topic', ?, 'c', ?)", + ); + insertDoc.run("doc-in", "In topic", "vectors"); + insertDoc.run("doc-out", "Other", null); + const insertChunk = vdb.prepare( + "INSERT INTO chunks (id, document_id, content, chunk_index) VALUES (?, ?, 'text', 0)", + ); + insertChunk.run("chunk-in", "doc-in"); + insertChunk.run("chunk-out", "doc-out"); + const insertVec = vdb.prepare( + "INSERT INTO chunk_embeddings (chunk_id, embedding) VALUES (?, x'00')", + ); + insertVec.run("chunk-in"); + insertVec.run("chunk-out"); + + deleteTopic(vdb, "vectors", { deleteDocuments: true }); + + const remaining = vdb.prepare("SELECT chunk_id FROM chunk_embeddings").all(); + expect(remaining).toEqual([{ chunk_id: "chunk-out" }]); + vdb.close(); + }); + + it("should delete documents when the vector table is missing", () => { + createTopic(db, { name: "NoVec" }); + db.prepare( + "INSERT INTO documents (id, source_type, title, content, topic_id) VALUES ('d', 'topic', 'D', 'c', 'novec')", + ).run(); + + deleteTopic(db, "novec", { deleteDocuments: true }); + + expect(db.prepare("SELECT id FROM documents WHERE id = 'd'").get()).toBeUndefined(); + }); + it("should set child topic parent_id to null on deletion", () => { const parent = createTopic(db, { name: "Parent" }); createTopic(db, { name: "Child", parentId: parent.id }); @@ -180,7 +217,7 @@ describe("topics", () => { ).run("d2", "topic", "Doc B", "content B", "docs", "manual"); const docs = getDocumentsByTopic(db, "docs"); - expect(docs.length).toBe(2); + expect(docs).toHaveLength(2); expect(docs[0]!.title).toBeDefined(); }); @@ -193,19 +230,19 @@ describe("topics", () => { } const page1 = getDocumentsByTopic(db, "paged", { limit: 2, offset: 0 }); - expect(page1.length).toBe(2); + expect(page1).toHaveLength(2); const page2 = getDocumentsByTopic(db, "paged", { limit: 2, offset: 2 }); - expect(page2.length).toBe(2); + expect(page2).toHaveLength(2); const page3 = getDocumentsByTopic(db, "paged", { limit: 2, offset: 4 }); - expect(page3.length).toBe(1); + expect(page3).toHaveLength(1); }); it("should return empty array for topic with no documents", () => { createTopic(db, { name: "Empty" }); const docs = getDocumentsByTopic(db, "empty"); - expect(docs.length).toBe(0); + expect(docs).toHaveLength(0); }); it("should throw TopicNotFoundError for nonexistent topic", () => { @@ -225,7 +262,7 @@ describe("topics", () => { ).run("s2", "topic", "Doc 2", "content", "alpha", "manual"); const stats = getTopicStats(db); - expect(stats.length).toBe(2); + expect(stats).toHaveLength(2); const alpha = stats.find((s) => s.id === "alpha"); const beta = stats.find((s) => s.id === "beta"); expect(alpha!.documentCount).toBe(2); @@ -234,7 +271,7 @@ describe("topics", () => { it("should return empty array when no topics exist", () => { const stats = getTopicStats(db); - expect(stats.length).toBe(0); + expect(stats).toHaveLength(0); }); }); }); diff --git a/tests/unit/update-document.test.ts b/tests/unit/update-document.test.ts index 61f9e7e2..4c1628de 100644 --- a/tests/unit/update-document.test.ts +++ b/tests/unit/update-document.test.ts @@ -31,13 +31,26 @@ describe("updateDocument", () => { docId = result.id; }); - it("should update title without re-chunking", async () => { + it("should update title without re-chunking, re-embedding the existing chunks", async () => { + const chunkIdsBefore = db + .prepare("SELECT id FROM chunks WHERE document_id = ? ORDER BY chunk_index") + .all(docId); const batchCallsBefore = provider.embedBatchCallCount; const input: UpdateDocumentInput = { title: "New Title" }; const updated: Document = await updateDocument(db, provider, docId, input); expect(updated.title).toBe("New Title"); expect(updated.content).toBe(baseInput.content); + // The title is part of the embedding text, so vectors are refreshed in place. + expect(provider.embedBatchCallCount).toBe(batchCallsBefore + 1); + expect( + db.prepare("SELECT id FROM chunks WHERE document_id = ? ORDER BY chunk_index").all(docId), + ).toEqual(chunkIdsBefore); + }); + + it("should not re-embed when only url or topic changes", async () => { + const batchCallsBefore = provider.embedBatchCallCount; + await updateDocument(db, provider, docId, { metadata: { url: "https://other.example" } }); expect(provider.embedBatchCallCount).toBe(batchCallsBefore); }); diff --git a/tests/unit/url-fetcher.test.ts b/tests/unit/url-fetcher.test.ts index 59f340cb..5c956e64 100644 --- a/tests/unit/url-fetcher.test.ts +++ b/tests/unit/url-fetcher.test.ts @@ -94,6 +94,20 @@ describe("fetchAndConvert", () => { expect(result.content).toContain("World"); }); + it("decodes HTML character references in the ", async () => { + const html = + "<html><head><title>textwrap — Text & wrapping

x

"; + mockFetch.mockResolvedValue({ + ok: true, + headers: new Headers({ "content-type": "text/html" }), + body: bodyStream(html), + text: () => Promise.resolve(html), + }); + + const result = await fetchAndConvert("https://example.com/page"); + expect(result.title).toBe("textwrap — Text & wrapping"); + }); + it("should return markdown/plain text as-is", async () => { const md = "# Markdown Title\n\nSome content here."; mockFetch.mockResolvedValue({ diff --git a/tests/unit/watcher-document.test.ts b/tests/unit/watcher-document.test.ts new file mode 100644 index 00000000..e269a382 --- /dev/null +++ b/tests/unit/watcher-document.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect, vi } from "vitest"; +import type Database from "better-sqlite3"; +import type { EmbeddingProvider } from "../../src/providers/embedding.js"; + +vi.mock("node:fs", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + watch: vi.fn(() => ({ on: vi.fn(), close: vi.fn() })), + readFileSync: vi.fn(() => "# Doc\n\nBody"), + statSync: vi.fn(() => ({ isFile: (): boolean => true })), + }; +}); + +vi.mock("../../src/core/indexing.js", async (importOriginal) => { + const original = await importOriginal(); + return { ...original, indexDocument: vi.fn(() => Promise.resolve({ id: "d1", chunkCount: 1 })) }; +}); + +const { watch } = await import("node:fs"); +const { indexDocument } = await import("../../src/core/indexing.js"); +const { FileWatcher } = await import("../../src/core/watcher.js"); + +describe("FileWatcher document metadata", () => { + it("indexes changed files with the given topic, library and source type", async () => { + vi.useFakeTimers(); + const db = { + prepare: vi.fn(() => ({ get: vi.fn(() => undefined), run: vi.fn() })), + } as unknown as Database.Database; + const onIndex = vi.fn(); + const watcher = new FileWatcher(db, {} as EmbeddingProvider, { + directory: "/docs", + extensions: [".md"], + document: { topicId: "t1", library: "lib", version: "2" }, + onIndex, + }); + watcher.start(); + const callback = vi.mocked(watch).mock.calls[0]?.[2] as (e: string, f: string) => void; + callback("change", "guide.md"); + await vi.advanceTimersByTimeAsync(500); + vi.useRealTimers(); + + expect(indexDocument).toHaveBeenCalledWith( + db, + expect.anything(), + expect.objectContaining({ + title: "guide", + sourceType: "library", + library: "lib", + version: "2", + topicId: "t1", + url: "/docs/guide.md", + }), + ); + expect(onIndex).toHaveBeenCalledWith("/docs/guide.md"); + watcher.stop(); + }); +}); diff --git a/tests/unit/watcher.test.ts b/tests/unit/watcher.test.ts index 394be10a..d5e5b032 100644 --- a/tests/unit/watcher.test.ts +++ b/tests/unit/watcher.test.ts @@ -73,8 +73,39 @@ describe("FileWatcher", () => { vi.restoreAllMocks(); }); - it("should export default watch extensions", () => { - expect(DEFAULT_WATCH_EXTENSIONS).toEqual([".md", ".mdx", ".txt", ".rst"]); + it("should derive default watch extensions from the parser registry", () => { + expect(DEFAULT_WATCH_EXTENSIONS).toEqual( + expect.arrayContaining([".md", ".mdx", ".txt", ".rst", ".pdf", ".docx"]), + ); + }); + + it("should skip files whose extension has no parser without erroring", async () => { + const db = createMockDb(); + const provider = createMockProvider(); + const onIndex = vi.fn(); + const onError = vi.fn(); + + mockStatSync.mockReturnValue({ isFile: () => true } as import("node:fs").Stats); + + const watcher = new FileWatcher(db, provider, { + directory: "/tmp/docs", + extensions: [".rs"], + onIndex, + onError, + }); + + watcher.start(); + watchCallback("change", "main.rs"); + + vi.advanceTimersByTime(500); + await vi.advanceTimersByTimeAsync(0); + + expect(mockStatSync).toHaveBeenCalledTimes(1); + expect(mockReadFileSync).not.toHaveBeenCalled(); + expect(onIndex).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + + watcher.stop(); }); it("should filter files by extension", () => { diff --git a/tests/unit/web-server.test.ts b/tests/unit/web-server.test.ts index 4342b6aa..703a1d50 100644 --- a/tests/unit/web-server.test.ts +++ b/tests/unit/web-server.test.ts @@ -5,6 +5,7 @@ import { createTestDb } from "../fixtures/test-db.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; import { insertDoc, insertChunk } from "../fixtures/helpers.js"; import { startWebServer, stopWebServer } from "../../src/web/server.js"; +import { testConfig } from "./operations/helpers.js"; let db: Database.Database; let provider: MockEmbeddingProvider; @@ -35,7 +36,10 @@ describe("web server", () => { insertDoc(db, "doc-2", "Node Guide"); insertChunk(db, "c2", "doc-2", "Node.js runtime for server-side JavaScript"); - server = await startWebServer(db, provider, { port: 0, host: "127.0.0.1" }); + server = await startWebServer( + { db, provider, config: testConfig() }, + { port: 0, host: "127.0.0.1" }, + ); const addr = server.address(); const port = typeof addr === "object" && addr ? addr.port : 3377; baseUrl = `http://127.0.0.1:${port}`; @@ -67,7 +71,7 @@ describe("web server", () => { expect(status).toBe(200); const docs = body as unknown as Array<{ title: string }>; expect(Array.isArray(docs)).toBe(true); - expect(docs.length).toBe(2); + expect(docs).toHaveLength(2); expect(docs[0]!.title).toBeDefined(); }); @@ -81,7 +85,7 @@ describe("web server", () => { it("GET /api/documents/:id returns 404 for missing document", async () => { const { status, body } = await fetchJson("/api/documents/nonexistent"); expect(status).toBe(404); - expect(body.error).toBe("Document not found"); + expect(body.error).toMatchObject({ code: "DOCUMENT_NOT_FOUND" }); }); it("GET /api/topics returns topic list with stats", async () => { @@ -89,7 +93,7 @@ describe("web server", () => { expect(status).toBe(200); const topics = body as unknown as Array<{ id: string; documentCount: number }>; expect(Array.isArray(topics)).toBe(true); - expect(topics.length).toBe(1); + expect(topics).toHaveLength(1); expect(topics[0]!.id).toBe("ts"); expect(topics[0]!.documentCount).toBe(1); }); @@ -105,13 +109,13 @@ describe("web server", () => { it("GET /api/search without q returns 400", async () => { const { status, body } = await fetchJson("/api/search"); expect(status).toBe(400); - expect(body.error).toContain("Missing"); + expect(body.error).toMatchObject({ code: "VALIDATION_ERROR" }); }); it("returns 404 for unknown routes", async () => { const { status, body } = await fetchJson("/api/unknown"); expect(status).toBe(404); - expect(body.error).toBe("Not found"); + expect(body.error).toMatchObject({ code: "NOT_FOUND" }); }); it("DELETE /api/documents/:id deletes a document", async () => { @@ -124,8 +128,57 @@ describe("web server", () => { expect(s2).toBe(404); }); - it("sets CORS headers", async () => { - const res = await fetch(`${baseUrl}/api/stats`); - expect(res.headers.get("access-control-allow-origin")).toBe("*"); + it("GET /graph returns the graph page and /api/graph its data", async () => { + const page = await fetch(`${baseUrl}/graph`); + expect(page.status).toBe(200); + expect(page.headers.get("content-type")).toContain("text/html"); + const { status, body } = await fetchJson("/api/graph?threshold=0.85&maxNodes=200&topic=ts"); + expect(status).toBe(200); + const nodes = (body as { nodes: Array<{ id: string; type: string }> }).nodes; + expect(nodes.filter((n) => n.type === "document").map((n) => n.id)).toEqual(["doc-1"]); + }); + + it("pages and filters the document list the way the dashboard asks", async () => { + const { body } = await fetchJson("/api/documents?limit=20&offset=0&topic=ts"); + const docs = body as unknown as Array<{ id: string; sourceType: string; updatedAt: string }>; + expect(docs.map((d) => d.id)).toEqual(["doc-1"]); + expect(docs[0]!.updatedAt).toBeDefined(); + const { body: page2 } = await fetchJson("/api/documents?limit=1&offset=1"); + expect(page2 as unknown as unknown[]).toHaveLength(1); + }); + + it("sends no CORS allow-origin to other origins and no CSP", async () => { + const res = await fetch(`${baseUrl}/api/stats`, { headers: { Origin: "http://evil.example" } }); + expect(res.status).toBe(200); + expect(res.headers.get("access-control-allow-origin")).toBeNull(); + expect(res.headers.get("content-security-policy")).toBeNull(); + expect(res.headers.get("x-frame-options")).toBe("DENY"); + }); + + it("rejects a cross-origin DELETE and its preflight grants nothing", async () => { + insertDoc(db, "doc-keep", "Keep me"); + const preflight = await fetch(`${baseUrl}/api/documents/doc-keep`, { + method: "OPTIONS", + headers: { Origin: "http://evil.example", "Access-Control-Request-Method": "DELETE" }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get("access-control-allow-origin")).toBeNull(); + + const { status, body } = await fetchJson("/api/documents/doc-keep", { + method: "DELETE", + headers: { Origin: "http://evil.example" }, + }); + expect(status).toBe(403); + expect(body.error).toMatchObject({ code: "FORBIDDEN_ORIGIN" }); + expect((await fetchJson("/api/documents/doc-keep")).status).toBe(200); + }); + + it("allows a same-origin DELETE", async () => { + insertDoc(db, "doc-same", "Same origin"); + const { status } = await fetchJson("/api/documents/doc-same", { + method: "DELETE", + headers: { Origin: baseUrl }, + }); + expect(status).toBe(200); }); }); diff --git a/tests/unit/webhook-events.test.ts b/tests/unit/webhook-events.test.ts new file mode 100644 index 00000000..0da1c665 --- /dev/null +++ b/tests/unit/webhook-events.test.ts @@ -0,0 +1,264 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { IncomingMessage, ServerResponse } from "node:http"; +import { Socket } from "node:net"; +import type Database from "better-sqlite3"; +import { createTestDbWithVec } from "../fixtures/test-db.js"; +import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; +import { initLogger } from "../../src/logger.js"; +import { ValidationError } from "../../src/errors.js"; +import { testConfig } from "./operations/helpers.js"; + +const resolve4 = vi.fn(); +vi.mock("node:dns", async (importOriginal) => { + const actual: typeof import("node:dns") = await importOriginal(); + return { + ...actual, + promises: { + ...actual.promises, + resolve4: (host: string): Promise => resolve4(host) as Promise, + resolve6: vi.fn().mockResolvedValue([]), + }, + }; +}); + +const { emitEvent, onEvent } = await import("../../src/core/events.js"); +const { createWebhook, deliverWebhook, signPayload } = await import("../../src/core/webhooks.js"); +const { indexDocument } = await import("../../src/core/indexing.js"); +const { updateDocument, deleteDocument } = await import("../../src/core/documents.js"); +const { rateDocument } = await import("../../src/core/ratings.js"); +const { searchDocuments } = await import("../../src/core/search.js"); +const { handleRequest } = await import("../../src/api/routes.js"); +const { createOperationContext } = await import("../../src/core/operations/index.js"); + +interface SentPayload { + event: string; + data: Record; +} + +function sentPayloads(mockFetch: ReturnType): SentPayload[] { + return mockFetch.mock.calls.map((c) => { + const init = c[1] as RequestInit; + return JSON.parse(init.body as string) as SentPayload; + }); +} + +describe("core events", () => { + beforeEach(() => { + initLogger("silent"); + }); + + it("delivers events to listeners until unsubscribed", () => { + const db = createTestDbWithVec(); + const listener = vi.fn(); + const off = onEvent(listener); + emitEvent(db, "document.deleted", { documentId: "d1" }); + off(); + emitEvent(db, "document.deleted", { documentId: "d2" }); + expect(listener).toHaveBeenCalledOnce(); + expect(listener).toHaveBeenCalledWith(db, "document.deleted", { documentId: "d1" }); + db.close(); + }); + + it("never throws into the caller when a listener throws or rejects", async () => { + const db = createTestDbWithVec(); + const offThrow = onEvent(() => { + throw new Error("sync boom"); + }); + const rejected = vi.fn().mockRejectedValue(new Error("async boom")); + const offReject = onEvent(rejected); + const after = vi.fn(); + const offAfter = onEvent(after); + expect(() => emitEvent(db, "search.executed", { query: "q" })).not.toThrow(); + await Promise.resolve(); + expect(rejected).toHaveBeenCalledOnce(); + expect(after).toHaveBeenCalledOnce(); + offThrow(); + offReject(); + offAfter(); + db.close(); + }); +}); + +describe("webhook delivery wiring", () => { + let db: Database.Database; + let provider: MockEmbeddingProvider; + let mockFetch: ReturnType; + + beforeEach(() => { + initLogger("silent"); + resolve4.mockResolvedValue(["93.184.216.34"]); + db = createTestDbWithVec(); + provider = new MockEmbeddingProvider(); + mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" }); + vi.stubGlobal("fetch", mockFetch); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + delete process.env["LIBSCOPE_SECRET_KEY"]; + db.close(); + }); + + it("fires document.created, updated, rated, deleted and search.executed", async () => { + await createWebhook(db, "https://example.com/hook", [ + "document.created", + "document.updated", + "document.deleted", + "document.rated", + "search.executed", + ]); + + const doc = await indexDocument(db, provider, { + title: "Webhook Doc", + content: "Some content about webhooks firing.", + sourceType: "manual", + library: "lib", + }); + await updateDocument(db, provider, doc.id, { title: "Webhook Doc v2" }); + rateDocument(db, { documentId: doc.id, rating: 4, feedback: "good" }); + await searchDocuments(db, provider, { query: "webhooks" }); + deleteDocument(db, doc.id); + + await vi.waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(5)); + const byEvent = new Map(sentPayloads(mockFetch).map((p) => [p.event, p.data])); + expect(byEvent.get("document.created")).toEqual({ + documentId: doc.id, + title: "Webhook Doc", + library: "lib", + }); + expect(byEvent.get("document.updated")).toMatchObject({ + documentId: doc.id, + title: "Webhook Doc v2", + }); + expect(byEvent.get("document.rated")).toEqual({ + documentId: doc.id, + rating: 4, + feedback: "good", + }); + expect(byEvent.get("search.executed")).toMatchObject({ query: "webhooks" }); + expect(byEvent.get("document.deleted")).toEqual({ documentId: doc.id }); + }); + + it("only delivers subscribed events and does not block or fail the mutation", async () => { + await createWebhook(db, "https://example.com/hook", ["document.deleted"]); + mockFetch.mockRejectedValue(new Error("network down")); + + const doc = await indexDocument(db, provider, { + title: "Quiet", + content: "No created hook for this one.", + sourceType: "manual", + }); + expect(mockFetch).not.toHaveBeenCalled(); + + expect(() => deleteDocument(db, doc.id)).not.toThrow(); + await vi.waitFor(() => { + const row = db.prepare("SELECT failure_count FROM webhooks").get() as { + failure_count: number; + }; + expect(row.failure_count).toBe(1); + }); + expect(sentPayloads(mockFetch)[0]!.event).toBe("document.deleted"); + }); + + it("does not throw into the mutation when the webhooks table is missing", async () => { + db.exec("DROP TABLE webhooks"); + await expect( + indexDocument(db, provider, { title: "T", content: "Body text", sourceType: "manual" }), + ).resolves.toMatchObject({ chunkCount: 1 }); + }); +}); + +describe("deliverWebhook", () => { + let db: Database.Database; + let mockFetch: ReturnType; + + beforeEach(() => { + initLogger("silent"); + resolve4.mockResolvedValue(["93.184.216.34"]); + db = createTestDbWithVec(); + mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 202, statusText: "Accepted" }); + vi.stubGlobal("fetch", mockFetch); + process.env["LIBSCOPE_SECRET_KEY"] = "test-master-key"; + }); + + afterEach(() => { + vi.unstubAllGlobals(); + delete process.env["LIBSCOPE_SECRET_KEY"]; + db.close(); + }); + + it("signs with the decrypted secret and refuses redirects", async () => { + const hook = await createWebhook( + db, + "https://example.com/hook", + ["document.created"], + "s3cr3t", + ); + expect(hook.secret).not.toBe("s3cr3t"); + + const resp = await deliverWebhook(hook, "document.created", { test: true }); + expect(resp.status).toBe(202); + + const init = mockFetch.mock.calls[0]![1] as RequestInit & { headers: Record }; + expect(init.redirect).toBe("error"); + expect(init.headers["X-LibScope-Signature"]).toBe(signPayload(init.body as string, "s3cr3t")); + }); + + it("rejects URLs that now resolve to a private IP without sending", async () => { + const hook = await createWebhook(db, "https://example.com/hook", ["document.created"]); + resolve4.mockResolvedValue(["10.0.0.5"]); + await expect(deliverWebhook(hook, "document.created", {})).rejects.toThrow(ValidationError); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it("REST test route signs with the decrypted secret and checks SSRF", async () => { + const hook = await createWebhook( + db, + "https://example.com/hook", + ["document.created"], + "s3cr3t", + ); + + const send = async (): Promise<{ status: number; body: string }> => { + const req = new IncomingMessage(new Socket()); + req.method = "POST"; + req.url = `/api/v1/webhooks/${hook.id}/test`; + req.headers = { host: "localhost:3378" }; + process.nextTick(() => { + req.push(null); + }); + const res = new ServerResponse(new IncomingMessage(new Socket())); + let body = ""; + res.end = ((chunk?: unknown) => { + if (typeof chunk === "string") body = chunk; + return res; + }) as typeof res.end; + await handleRequest( + req, + res, + createOperationContext({ + db, + provider: new MockEmbeddingProvider(), + config: testConfig(), + surface: "api", + }), + ); + return { status: res.statusCode, body }; + }; + + const ok = await send(); + expect(ok.status).toBe(200); + expect(JSON.parse(ok.body)).toMatchObject({ data: { status: 202 } }); + const init = mockFetch.mock.calls[0]![1] as RequestInit & { headers: Record }; + expect(init.headers["X-LibScope-Signature"]).toBe(signPayload(init.body as string, "s3cr3t")); + expect(JSON.parse(init.body as string)).toMatchObject({ + event: "document.created", + data: { test: true }, + }); + + resolve4.mockResolvedValue(["127.0.0.1"]); + const blocked = await send(); + expect(blocked.status).toBe(400); + expect(mockFetch).toHaveBeenCalledOnce(); + }); +}); diff --git a/tests/unit/webhooks.test.ts b/tests/unit/webhooks.test.ts index 1c371678..6e92d9c8 100644 --- a/tests/unit/webhooks.test.ts +++ b/tests/unit/webhooks.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, vi, afterEach } from "vitest"; import type Database from "better-sqlite3"; import { createTestDb } from "../fixtures/test-db.js"; import type { WebhookEvent } from "../../src/core/webhooks.js"; -import { ValidationError } from "../../src/errors.js"; +import { NotFoundError, ValidationError } from "../../src/errors.js"; import { initLogger } from "../../src/logger.js"; // Mock DNS resolution so webhook SSRF checks pass in tests @@ -139,8 +139,8 @@ describe("webhooks", () => { expect(fetched.url).toBe(created.url); }); - it("should throw ValidationError for non-existent id", () => { - expect(() => getWebhook(db, "non-existent")).toThrow(ValidationError); + it("should throw NotFoundError for non-existent id", () => { + expect(() => getWebhook(db, "non-existent")).toThrow(NotFoundError); }); }); @@ -151,8 +151,8 @@ describe("webhooks", () => { expect(listWebhooks(db)).toHaveLength(0); }); - it("should throw ValidationError for non-existent id", () => { - expect(() => deleteWebhook(db, "non-existent")).toThrow(ValidationError); + it("should throw NotFoundError for non-existent id", () => { + expect(() => deleteWebhook(db, "non-existent")).toThrow(NotFoundError); }); }); @@ -215,9 +215,9 @@ describe("webhooks", () => { ).rejects.toThrow(ValidationError); }); - it("should throw ValidationError for non-existent id", async () => { + it("should throw NotFoundError for non-existent id", async () => { await expect(updateWebhook(db, "non-existent", { url: "https://new.com" })).rejects.toThrow( - ValidationError, + NotFoundError, ); }); }); diff --git a/vitest.config.ts b/vitest.config.ts index f48df703..17f70b53 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,18 +10,15 @@ export default defineConfig({ include: ["src/**/*.ts"], exclude: [ "src/cli/**", - "src/mcp/server.ts", + "src/mcp/main.ts", "src/providers/local.ts", "src/providers/ollama.ts", "src/providers/openai.ts", - "src/api/indexing/repoConfig.ts", - "src/api/indexing/repoIndexer.ts", "src/db/connection.ts", "src/core/index.ts", "src/db/index.ts", "src/providers/index.ts", "src/providers/embedding.ts", - "src/web/graph-api.ts", ], thresholds: { statements: 75,