diff --git a/docs/guides/KNOWLEDGE_MIGRATION.md b/docs/guides/KNOWLEDGE_MIGRATION.md new file mode 100644 index 00000000..9836dd72 --- /dev/null +++ b/docs/guides/KNOWLEDGE_MIGRATION.md @@ -0,0 +1,117 @@ +# Mate Knowledge workflow and migration guide + +This guide describes the currently supported Mate Knowledge workflow. It is intentionally generic: narrative material belongs in guarded Markdown documents, while structured records belong in a Knowledge database. The import tools map external source rows into that database without accepting filesystem paths or caller-selected owners. + +## Supported tool surface + +Text documents: + +- `read_knowledge_document` reads current Markdown content and its stable `documentId` and `revision`. +- `create_knowledge_document` creates safe nested Markdown paths. +- `update_knowledge_document` replaces content only with the current `expectedRevision`. +- `list_knowledge_document_history` lists recoverable revisions. +- `restore_knowledge_document` restores a selected version with a current-revision precondition. + +Structured records: + +- `list_databases` discovers database summaries; summaries are not row content. +- `read_database` reads the selected database schema, views, revision, and count. +- `query_database` uses stable field ids, typed filters, explicit sorts, bounded pages, cursors, and filter-scoped count/numeric sums. +- `read_database_record` reads a row by stable database and record IDs. +- `create_database_record` and `update_database_record` use typed values, stable field IDs, revisions, and retry operation IDs. +- `archive_database_record`, `recover_database_record`, and `retry_database_sync` provide recoverable lifecycle and projection retry behavior. +- `import_database_records` imports up to 100 source rows or 256 KiB per batch with immutable source namespace/key/payload identity. +- `read_database_import_progress` returns bounded source-to-native record mappings for later ID-based corrections. + +Database list/search summaries are discovery projections, not row content. A workflow that needs row values must discover the database, read its schema, query with an explicit filter, and read individual records by returned IDs. Numeric totals must come from the query result for the same filter scope, never from a page count or an unfiltered search. Text search and text-document reads retain their own content and revision semantics. + +## Workflow recipes + +### Daily narrative capture + +1. Use `create_knowledge_document` for a new narrative or multiline Markdown document. +2. Preserve wiki links as text; use the returned `documentId` and `revision` for later edits. +3. On update, supply `expectedRevision`. If it is rejected, call `read_knowledge_document`, merge against the returned content, and retry with the new revision. +4. Treat thrown errors or error responses as failures. If a successful response includes a revision/id plus `indexSyncPending`, the canonical document is saved but indexing is pending; report that exact state, do not call it indexed, and do not blindly retry the mutation. `syncError` identifies the projection problem and is not permission to create a duplicate document. + +Example update shape: + +```json +{ + "documentId": "", + "content": "# Daily note\nSee [[areas/project-plan]].\n", + "expectedRevision": "" +} +``` + +### Structured follow-up or finance records + +1. Use `list_databases` to discover candidates, then `read_database` to inspect the schema and stable field IDs. +2. Use `query_database` with typed filters such as category, spender/contact, status, or date; provide explicit sorts and a bounded limit. +3. Use the returned count and numeric sums for the exact filter scope. Follow returned stable record IDs with `read_database_record` when full row content is needed. +4. For corrections, use `update_database_record` with the row revision and an operation ID. A rejected revision requires rereading the row before retrying. + +A follow-up cadence or date cutoff is caller-provided input. Knowledge tools do not schedule reminders or invent a cutoff. + +### Source-row import + +1. Discover and read the target database schema first. Map each source field to a stable database field id; preserve the original source payload in each row. +2. Call `import_database_records` with `databaseId`, the current `expectedDatabaseRevision`, a fresh `operationId`, `sourceNamespace`, and rows containing `sourceKey`, `sourcePayload`, and typed `values`. The import fingerprint covers both the immutable source payload and supplied values. +3. Represent record references with `type: "record"`, `sourceNamespace`, and `sourceKey`. Same-batch forward references are supported; cross-database references must already be imported in the current Mate. +4. On an uncertain response, replay the exact operation id and identical payload while its receipt is active. After receipt expiry, use a fresh operation id with the current database revision only after checking progress. A changed payload or values for an existing source key is a source conflict, not a correction; use `update_database_record` after resolving the returned native record id. +5. Use `read_database_import_progress` to map source keys to native record IDs, then query and correct records through normal revision-guarded CRUD. Human corrections are preserved by source replay because imported mappings are immutable. + +Import writes may return `projectionPending` or `projectionError`: the canonical database revision and records are saved, but the summary projection is pending or failed. Report that exact state and use `retry_database_sync` only for the projection; do not repeat the import blindly. + +### Persistence adapter boundary + +`createKnowledgeDbService({ storageAdapter })` accepts an adapter with `contractVersion: 1`. A synchronous `storageFactory({ projectDir })` is another injection route. Without injection, `knowledge-db-storage.js` selects the production default through `createDefaultKnowledgeDbStorage`; NeDB is loaded lazily. Future desktop storage selection belongs in this factory, without engine branches in domain or tool code. + +The minimal contract is `initialize()`, `hasStore()`, `isReady()`, `get(id)`, `list()`, `getCached()`, `insert(snapshot)`, `replaceIfRevision(id, expectedRevision, snapshot)`, and `serialize(scopeKey, work)`. Async operations return promises; initialization and insertion resolve without a native result. Snapshots are detached JSON aggregates with the existing stable `_id` and `revision` fields. `_id` is the legacy Knowledge identity, not a query operator; another engine may map it to its own primary key. Cached snapshots include only committed state. Optional `diagnostics()` returns backend-specific JSON through `readStorageStatus()`; an adapter without diagnostics returns `null` there. + +Conditional replacement atomically checks identity and revision and replaces the complete snapshot, returning `true` on replacement and `false` without writing on mismatch or absence. It allows an unchanged revision for receipt and projection-status writes. Insertion rejects duplicate identity with `ALREADY_EXISTS`. `serialize` excludes other callbacks with the same logical scope across facades of the same physical store. Its callback receives no native handle and can await adapter methods; it does not provide a multi-aggregate transaction. Engines must separately protect I/O, recovery and maintenance from interleaving. Do not recursively await another callback with the same scope. + +The service/import layer owns schemas, references, filtering/totals, revisions, history, archives, receipt windows, source fingerprints, atomic batch validation, the 20,000-record limit and the 16 MiB aggregate limit. The default adapter owns NeDB loading, filesystem guards, the unchanged `knowledge/.clay-db/databases.db` representation, shared lifecycle and queues, native conditional writes and maintenance. Summary callbacks remain outside native I/O gates but inside the database's logical mutation scope. Write errors can be ambiguous after an append; the adapter reloads committed state while holding the shared I/O gate, or invalidates the store if recovery fails. Exact operation receipts settle retries. A maintenance error cannot turn an already committed write into a mutation failure. + +This extraction keeps aggregate-oriented storage and scan-based domain queries; it does not implement SQLite, larger capacity, row-oriented storage or migration. `initialize()` no longer exposes a datastore, and storage filenames are absent from the service interface. Engine-specific tests use NeDB fixtures/diagnostics rather than consuming native handles from the service. The compatibility fixture was generated from unmodified `76abe088` source and retains records, receipts, history and import mappings. + +Adapter verification: + +```sh +node --test test/knowledge-db-storage-contract.test.js test/knowledge-db-storage-compatibility.test.js +``` + +### Storage maintenance and backup + +The import implementation uses a bounded datastore maintenance API. Reads, writes, recovery, and compaction are serialized; there are no background timers. Compaction is triggered when the journal exceeds the greater of 1 MiB or three times live data, or after 64 writes with at least 1 MiB reclaimable. A maintenance failure does not undo a successful canonical write; later writes retry maintenance, and persistent growth can reject the next write with `STORAGE_MAINTENANCE_BLOCKED`. `readStorageStatus` exposes internal live/journal bytes, peak bytes, compaction, recovery, and error state; it is not a public Mate tool. + +The accepted database test run imported 8,563 vocabulary rows and 537 finance rows, then verified isolated-process restart and backup replay with counts, references, and totals. The storage test run also covers all four storage failure/concurrency cases. Its measured 8,563-row run reported a 33,483,840-byte peak journal and 25,718,220-byte final journal. The independently verified 20,000-row run measured a 59,093,150-byte peak journal, 44,432,487-byte final journal, 14,811,046-byte aggregate, 14,811,047 live bytes, 132 compactions, and 200 canonical plus 200 projection writes. The normal configured summary path adds a second aggregate status write per batch. + +Verification commands: + +```sh +node --test test/knowledge-db-import.test.js test/knowledge-db-import-storage.test.js +``` + +Back up only with writes quiesced. Restore into a separate location and fresh process, then verify database counts, source references, and filter-scoped totals. Do not copy datastore files while mutations are active. A temporary compaction snapshot requires an additional live-data-sized disk copy. + +### Correction and recovery + +After a save, create a fresh service binding or process and verify that the document or record can be read by stable ID. For text, use `list_knowledge_document_history` and `restore_knowledge_document` with the current revision. For databases, use the record revision and recoverable archive tools. A foreign Mate, foreign user, shared/private-invalid session, or stale captured session must be refused. + +## Acceptance checklist + +- Create, fresh read, correction, and stale-write rejection work through actual session-bound handlers. +- Text wiki-link content survives agent edits and history restoration. +- Structured queries return expected filtered counts and sums, with pagination not mistaken for totals. +- Import handlers preserve source identity, replay exact operations, reject changed-source payloads, resolve supported references, and expose progress mappings. +- Stable IDs, not name guesses, are used for subsequent reads and updates. +- A newly bound service and the isolated restart/backup replay checks see saved state by stable IDs, references, and totals. +- Projection errors and pending states distinguish canonical save from indexing status; only actual errors are failures, and mutations are not retried blindly. +- Clay readiness includes the Driver-runtime check that Gmail and Google Calendar `get_profile` both succeeded. Target-Mate integration checks remain reserved for the actual migration; no Argo Mate is created here. + +## Limits and pending work + +Nested Markdown is bounded by the existing file-store name/content/history limits. Import batches are bounded at 100 rows and 256 KiB; the database aggregate is bounded at 20,000 records and 16 MiB including stored history, mappings, archives, and receipts. Up to 500 live mutation receipts are shared across import and ordinary writes within the 24-hour retry window. Fingerprints cover both the original `sourcePayload` and typed `values`; after receipt expiry, a fresh operation ID and current database revision are required. The 20,000-row ceiling is distinct from the aggregate byte ceiling: not every payload set fits within 16 MiB. Database schemas, record fields, query pages, filters, and retry receipts remain bounded by the existing database service. Text fields accept multiline strings, while narrative multiline content is usually clearer in text documents. + +The public import interface and bounded storage maintenance are implemented and covered by the cited synthetic tests. This documents user-scope readiness only; it does not claim deployed or live-migration readiness. Gmail and Google Calendar `get_profile` succeeded in the Driver runtime, while target-Mate integration checks remain reserved for the actual migration. No live migration or Argo Mate creation is performed here. diff --git a/docs/guides/MODULE_MAP.md b/docs/guides/MODULE_MAP.md index 061ef279..05cf4ce6 100644 --- a/docs/guides/MODULE_MAP.md +++ b/docs/guides/MODULE_MAP.md @@ -84,7 +84,7 @@ Favorites-as-tag behavior supersedes the older exclusive-folder wording above: F | Module | Message types | Concern | |--------|--------------|---------| -| `project-knowledge.js` + `knowledge-file-store.js` + `knowledge-db-service.js` + `knowledge-db-validation.js` | `knowledge_list`, `knowledge_search`, document messages, and `knowledge_db_*` | Canonical Mate-project Knowledge for exactly text and database items. Markdown remains portable files; first-class databases are bounded NeDB aggregate documents in `knowledge/.clay-db/databases.db` with typed stable field/record ids, revisions, recoverable archives/history, a bounded 24-hour idempotency window, saved Table views, and retryable common Knowledge summary projection. Client database rendering is isolated in `knowledge-db-workbench.js`, while immutable record attempts and complex-view presentation updates live in `knowledge-db-edit-state.js`; the normal pane is read-only and mutation forms are fullscreen. | +| `project-knowledge.js` + `knowledge-file-store.js` + `knowledge-db-service.js` + `knowledge-db-validation.js` + `knowledge-db-import.js` + `knowledge-db-import-validation.js` + `knowledge-db-storage.js` + `knowledge-db-storage-nedb.js` + `knowledge-db-storage-nedb-maintenance.js` | `knowledge_list`, `knowledge_search`, document messages, and `knowledge_db_*` | Canonical Mate-project Knowledge for exactly text and database items. Markdown remains portable files; first-class databases use an engine-neutral aggregate persistence contract with a lazy default NeDB adapter and bounded aggregate documents in `knowledge/.clay-db/databases.db` with typed stable field/record ids, revisions, recoverable archives/history, a bounded 24-hour idempotency window, saved Table views, retryable common Knowledge summary projection, guarded source-row import validation, and serialized bounded journal maintenance/recovery. Domain rules stay in the service/import modules; `knowledge-db-storage.js` validates adapter version 1 and selects the default centrally, `knowledge-db-storage-nedb.js` owns paths/lifecycle/shared scoped queues/native CAS, and its maintenance helper owns the shared I/O gate and journal policy. Inject `storageAdapter` or a synchronous `storageFactory`; initialization returns no engine handle. Client database rendering is isolated in `knowledge-db-workbench.js`, while immutable record attempts and complex-view presentation updates live in `knowledge-db-edit-state.js`; the normal pane is read-only and mutation forms are fullscreen. | | `project-session-github.js` + `session-github.js` | Session-bound GitHub work links (`clay-github` native/MCP/stdio tools) and `session_github_refresh`. Exact issue/PR URLs are explicitly associated by the Driver, verified with read-only authenticated `gh`, and persisted in session metadata. Uses per-OS-user credentials, live access checks, bounded refresh, and stale status on failures; never infers associations from a shared checkout or mutates GitHub. | | `project-sessions.js` | `new_session`, `switch_session`, `delete_session`, `rename_session`, `resume_session`, `fork_session`, `rewind_*`, `permission_response`, `elicitation_response`, `set_model`, `set_effort`, `set_thinking`, `set_betas`, `set_*_mode`, `browse_dir`, `add_project`, `create_project`, `clone_project`, `create_worktree`, `remove_project*`, `schedule_move`, `reorder_projects`, `set_project_title`, `set_project_icon`, `get_daemon_config`, `set_pin`, `set_keep_awake`, `set_auto_continue`, `set_image_retention`, `shutdown_server`, `restart_server`, `process_stats`, `stop`, `stop_task`, `kill_process`, `set_update_channel`, `check_update`, `update_now`, `ask_user_response`, `input_sync`, `cursor_*`, `text_select`, `push_subscribe`, `load_more_history`, `search_sessions`, `search_session_content`, `list_cli_sessions`, `set_session_visibility`, `transfer_project_owner`, `set_mate_dm` | Session lifecycle, config, project management, daemon settings, permissions, updates | | `project-session-delete.js` | `delete_session` and `bulk_delete_sessions` scope and authorization. Explicit human deletion of a Driver also deletes every Split Worker generation (current and replaced) that the provenance hierarchy links to it: a unique parent origin, the same owner, and a parent that is not itself a Worker. Ambiguous and cross-owner links are never expanded. The caller's delete permission and access are checked for the whole scope before anything happens. If any cascaded Worker is inaccessible, the request is refused. PTYs and title watchers are released, then the existing session-manager deletion cleanup runs. Deleting a Worker affects only that Worker; close, dissolve, and replacement never come through here and keep history | @@ -213,7 +213,7 @@ Favorites-as-tag behavior supersedes the older exclusive-folder wording above: F | `project-logs-versioning.js` | Project Driver review and revision control: `review` (incorporate writes exactly one record that both resolves the comment and carries the new snapshot; clarify/decline write no revision), `revert` (appends a new full-snapshot revision, never erases later history, refuses a no-op or a deleted entry), and `readRevision` | | `project-logs-schema.js` | Project Logs vocabulary and validation. **Category is a bounded project-local label, never a global enum**: `normalizeCategory` reduces free text to a lowercase hyphen-separated label of letters and digits in any script (Hangul, CJK, Cyrillic and accented Latin included), bounded at 32 code points, and refuses empty, path-shaped, or otherwise unusable input. A supplied malformed filter is an error, never a silently dropped one. `safeCategory` is the non-throwing form used only when reading a possibly corrupt stored record. Seed categories are tool guidance only. Priority stays the stable `normal`/`important`/`urgent` enum. Also holds the deterministic summary fallback and the shared text/shape normalizers | | `knowledge-search.js` | **Shared BM25 adapter.** Thin wrapper over `session-search.js`'s `buildIndex`/`searchIndex` and its CJK-bigram tokenizer. Field weighting is controlled repetition before indexing. Long records are segmented into overlapping bounded documents (short fields anchor every segment) and collapsed back to one result per item by best segment score, so tail matches are never lost to truncation. Coverage is bounded at `MAX_INDEXED_CHARS` per record and reported via `coverage()`, so an incompletely indexed record is stated rather than implied. Every Knowledge surface ranks through this; do not add a second ranker or embeddings. Also owns `compareIds()`, the locale-independent UTF-16 code-unit comparator every Knowledge surface must use to break score/recency ties on opaque base64url ids and refs; `localeCompare` orders `-` and `_` by ICU collation and is not deterministic across environments | -| `mate-knowledge-service.js` + `mate-knowledge-mcp-server.js` + `project-mate-knowledge.js` | Session-bound `clay-knowledge` MCP/native/bridge surface. An ordinary Mate gets text list/search/read plus database discovery, schema, Table view, query, record CRUD and recoverable archive tools bound to its own live project service with no owner/scope argument. Authoritative builtin Clay retains same-user cross-Mate read tools over the common projection. Non-Mate projects get nothing here. | +| `mate-knowledge-service.js` + `mate-knowledge-mate-bound.js` + `mate-knowledge-text-tools.js` + `mate-knowledge-mcp-server.js` + `project-mate-knowledge.js` | Session-bound `clay-knowledge` MCP/native/bridge surface. An ordinary Mate gets text discovery plus guarded nested-Markdown read/create/update/history/restore and database discovery, schema, typed filtered/sorted query, record CRUD, bounded source-row import/progress, and recoverable archive tools bound to its own live project service with no owner/scope/actor argument. Database list/search summaries are for discovery; import mappings and query results use stable IDs, and revision/projection status remains explicit. Authoritative builtin Clay retains same-user cross-Mate read tools over the common projection. Non-Mate projects get nothing here. | | `project-logs-query.js` | Project Logs filtering (exact project-local category, priority enum, tag), BM25 ranking with summary weighting, snippets, pagination, and the ledger row projection, extracted from `project-logs-store.js` to keep it under the size limit | | `knowledge-import.js` | Import-keyed projection over the record backend. Stable import keys make imports idempotent; tombstones handle logical removal; oversized sources are stored as content-addressed chunks and reassembled exactly (never truncated). Index is maintained incrementally so a no-op run stays cheap | | `mate-knowledge-sync.js` | **Single source-to-record mapping** for Mate Knowledge, used by both the startup migration and the live write-through bridge. `syncMateSource`/`reconcileSource` reconcile one legacy source: create, revise, revive, or tombstone. Journal reconciliation tombstones lines removed from the legacy file. Identity files and `sticky-notes.md` are never mirrored. Called from `project-knowledge.js`, `project-mate-interaction.js`, and `project-memory.js` after their legacy writes | diff --git a/lib/dm-digest-cursor.js b/lib/dm-digest-cursor.js new file mode 100644 index 00000000..a5a6f214 --- /dev/null +++ b/lib/dm-digest-cursor.js @@ -0,0 +1,120 @@ +var CONVERSATION_CAP = 6000; + +// SDK text is persisted as delta before onTurnDone; the response preview +// is only a notification excerpt. Historical complete text entries remain readable. +function isDigestEntry(entry) { + return entry && (entry.type === "user_message" || entry.type === "assistant_message" || entry.type === "delta" || entry.type === "text"); +} + +function copyCursor(cursor) { + return { + historyIndex: Math.max(0, cursor.historyIndex || 0), + textOffset: Math.max(0, cursor.textOffset || 0), + }; +} + +function readState(history) { + var state = { cursor: { historyIndex: 0, textOffset: 0 } }; + for (var i = history.length - 1; i >= 0; i--) { + var entry = history[i]; + if (!entry || entry.type !== "digest_checkpoint") continue; + if (entry.cursor && typeof entry.cursor.historyIndex === "number") { + state.cursor = copyCursor(entry.cursor); + } else if (typeof entry.digestedIndex === "number") { + // Legacy checkpoints pointed at the next history entry and had no + // within-entry progress. Treat them as fully consumed entries. + state.cursor = { historyIndex: Math.max(0, entry.digestedIndex), textOffset: 0 }; + } + break; + } + return state; +} + +function readCursor(history) { + return readState(history).cursor; +} + +function sameCursor(left, right) { + return left.historyIndex === right.historyIndex && left.textOffset === right.textOffset; +} + +function safeTextEnd(text, start, maximum) { + var end = Math.min(text.length, start + maximum); + if (end < text.length && end > start) { + var previous = text.charCodeAt(end - 1); + if (previous >= 0xD800 && previous <= 0xDBFF) end--; + } + // The normal cap leaves enough room for a code point. Keep this fallback + // defensive so a future smaller cap cannot make the cursor stall. + if (end === start && end < text.length) { + end = start + (text.charCodeAt(start) >= 0xD800 && text.charCodeAt(start) <= 0xDBFF ? 2 : 1); + } + return Math.min(end, text.length); +} + +function collectChunk(history, startCursor) { + var cursor = copyCursor(startCursor); + var parts = []; + var totalLength = 0; + var hadHistoryProgress = false; + var partCount = 0; + + function appendPart(value) { + var separatorLength = partCount > 0 ? 1 : 0; + if (totalLength + separatorLength + value.length > CONVERSATION_CAP) return false; + parts.push(value); + totalLength += separatorLength + value.length; + partCount++; + return true; + } + + while (cursor.historyIndex < history.length) { + var entry = history[cursor.historyIndex]; + if (!isDigestEntry(entry) || typeof entry.text !== "string" || !entry.text) { + cursor.historyIndex++; + cursor.textOffset = 0; + hadHistoryProgress = true; + continue; + } + + var text = entry.text; + var offset = Math.min(cursor.textOffset, text.length); + + var label = entry.type === "user_message" ? "User: " : "Mate: "; + var available = CONVERSATION_CAP - totalLength - (partCount > 0 ? 1 : 0) - label.length; + if (available <= 0 && partCount > 0) break; + if (available <= 0) available = CONVERSATION_CAP - label.length; + + if (offset < text.length) { + var end = safeTextEnd(text, offset, available); + var value = label + text.substring(offset, end); + if (!appendPart(value)) break; + cursor.textOffset = end; + hadHistoryProgress = true; + if (end < text.length) break; + } + + cursor.historyIndex++; + cursor.textOffset = 0; + } + + return { + conversationContent: parts.join("\n"), + cursor: cursor, + hasHistoryProgress: hadHistoryProgress, + }; +} + +function hasPending(history, cursor) { + var probe = collectChunk(history, cursor, ""); + return !!probe.conversationContent; +} + +module.exports = { + CONVERSATION_CAP: CONVERSATION_CAP, + collectChunk: collectChunk, + hasPending: hasPending, + readCursor: readCursor, + readState: readState, + sameCursor: sameCursor, +}; diff --git a/lib/knowledge-db-import-validation.js b/lib/knowledge-db-import-validation.js new file mode 100644 index 00000000..0bb3c5b4 --- /dev/null +++ b/lib/knowledge-db-import-validation.js @@ -0,0 +1,46 @@ +var crypto = require("crypto"); +var validation = require("./knowledge-db-validation"); +var MAX_BATCH_ROWS = 100; +var MAX_BATCH_BYTES = 256 * 1024; + +function fail(message, code) { throw Object.assign(new Error(message), { code: code || "INVALID" }); } +function json(value, depth) { + if (depth > 32) fail("Import JSON exceeds 32 nesting levels."); + if (value === null || typeof value === "string" || typeof value === "boolean") return; + if (typeof value === "number" && Number.isFinite(value)) return; + if (typeof value !== "object" || !value) fail("Import payload must contain finite JSON values."); + if (!Array.isArray(value)) validation.plainObject(value, "Import payload"); + Object.keys(value).forEach(function (key) { json(value[key], depth + 1); }); +} +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== "object") return value; + var out = {}; Object.keys(value).sort().forEach(function (key) { out[key] = canonical(value[key]); }); return out; +} +function identity(namespace, key) { return JSON.stringify([namespace, key]); } +function sourceText(value, label, max) { + var clean = validation.text(value, label, max, false); + if (clean !== value) fail(label + " must not have surrounding whitespace."); + return clean; +} +function batch(args) { + validation.plainObject(args, "Import request"); + var namespace = sourceText(args.sourceNamespace, "Source namespace", 120); + if (!Array.isArray(args.rows) || !args.rows.length || args.rows.length > MAX_BATCH_ROWS) fail("Import batch must contain between 1 and " + MAX_BATCH_ROWS + " rows.", "IMPORT_BATCH_LIMIT"); + json(args.rows, 0); + if (Buffer.byteLength(JSON.stringify(args.rows), "utf8") > MAX_BATCH_BYTES) fail("Import rows exceed the " + MAX_BATCH_BYTES + " byte input limit.", "IMPORT_BYTES_LIMIT"); + var seen = new Set(); + var rows = args.rows.map(function (row) { + validation.plainObject(row, "Import row"); + var key = sourceText(row.sourceKey, "Source row key", 200); + if (seen.has(key)) fail("Import batch contains duplicate source row keys."); + seen.add(key); + if (!Object.prototype.hasOwnProperty.call(row, "sourcePayload")) fail("Each import row needs an immutable sourcePayload."); + validation.plainObject(row.values, "Import values"); + var fingerprint = crypto.createHash("sha256").update(JSON.stringify(canonical({ sourcePayload: row.sourcePayload, values: row.values }))).digest("hex"); + return { sourceKey: key, fingerprint: fingerprint, values: row.values }; + }); + return { sourceNamespace: namespace, rows: rows }; +} + +module.exports = { batch: batch, identity: identity, sourceText: sourceText, MAX_BATCH_ROWS: MAX_BATCH_ROWS, MAX_BATCH_BYTES: MAX_BATCH_BYTES }; diff --git a/lib/knowledge-db-import.js b/lib/knowledge-db-import.js new file mode 100644 index 00000000..77158d44 --- /dev/null +++ b/lib/knowledge-db-import.js @@ -0,0 +1,98 @@ +var validation = require("./knowledge-db-validation"); +var input = require("./knowledge-db-import-validation"); + +// importRecords({ databaseId, expectedDatabaseRevision, operationId, sourceNamespace, +// rows: [{ sourceKey, sourcePayload, values }], actor }) creates only unseen source rows. +// Both sourcePayload and supplied values are immutable import identity; corrections use CRUD. +// Record references can use sourceNamespace/sourceKey instead of id, including forward +// references within this batch. Cross-database sources must already be imported locally. +// readImportProgress({ databaseId, sourceNamespace, offset?, limit? }) returns native ids. +// Import mappings survive receipt expiry; replay with a fresh operation id and current revision. +// Limits: 100 rows / 256 KiB of row JSON per batch; shared 20000-record / 16 MiB +// aggregate bounds include archives, history, mappings and receipts (500 live / 24 hours). +// Each fresh operation retains one receipt and uses one canonical replacement; source +// replay preserves the revision. New rows also use the existing optional summary +// projection write. No filesystem input is accepted. +function attachKnowledgeDbImport(ctx) { + function mappings(database) { return database.importMappings || []; } + function index(database) { + var out = new Map(); mappings(database).forEach(function (mapping) { out.set(input.identity(mapping.sourceNamespace, mapping.sourceKey), mapping); }); return out; + } + function convert(values, schema, databases, next, planned) { + var fields = {}; schema.forEach(function (field) { fields[field.id] = field; }); + var out = Object.assign({}, values); + Object.keys(out).forEach(function (fieldId) { + var field = fields[fieldId]; var value = out[fieldId]; + if (!field || field.type !== "reference" || !value || !Object.prototype.hasOwnProperty.call(value, "sourceKey")) return; + validation.plainObject(value, "Import reference"); + if (value.type !== "record" || value.id !== undefined) throw ctx.error("Source references require type record and no native id.", "INVALID"); + var databaseId = validation.id(value.databaseId || field.targetDatabaseId || next._id, "reference database id"); + var namespace = input.sourceText(value.sourceNamespace, "Reference source namespace", 120); + var key = input.sourceText(value.sourceKey, "Reference source key", 200); + var target = databases.find(function (database) { return database._id === databaseId; }); + var mapping = databaseId === next._id ? planned.get(input.identity(namespace, key)) : target && index(target).get(input.identity(namespace, key)); + if (!mapping) throw ctx.error("Reference source row is outside this Mate or has not been imported.", "FORBIDDEN_REFERENCE"); + out[fieldId] = { type: "record", databaseId: databaseId, id: mapping.recordId }; + }); + return out; + } + function importRecords(args) { + // Snapshot caller-owned JSON before any asynchronous operation. + var batch = ctx.clone(input.batch(args)); args = ctx.clone(args); + var operationId = ctx.mutationId(args.operationId); var fingerprint = ctx.fingerprint("import", args); + return ctx.serialize(args.databaseId, function () { return ctx.load(args.databaseId).then(async function (current) { + var duplicate = ctx.receipt(current, operationId, fingerprint); + if (duplicate) return Object.assign({}, duplicate, { projectionPending: current.projectionPending === true, projectionError: current.projectionError || null }); + if (current.archivedAt) throw ctx.error("Knowledge database is archived.", "ARCHIVED", current.revision); + ctx.ensureRevision(args.expectedDatabaseRevision, current.revision, "database"); + var next = ctx.clone(current); var planned = index(current); var added = []; + var resultMappings = batch.rows.map(function (row) { + var identity = input.identity(batch.sourceNamespace, row.sourceKey); var existing = planned.get(identity); + if (existing && existing.fingerprint !== row.fingerprint) throw ctx.error("Source row was already imported with a different payload: " + row.sourceKey, "IMPORT_SOURCE_CONFLICT"); + if (existing) return Object.assign({}, existing, { created: false }); + var mapping = { sourceNamespace: batch.sourceNamespace, sourceKey: row.sourceKey, fingerprint: row.fingerprint, recordId: ctx.newId("rec") }; + planned.set(identity, mapping); added.push({ row: row, mapping: mapping }); + return Object.assign({}, mapping, { created: true }); + }); + if (next.records.length + added.length > validation.MAX_RECORDS) throw ctx.error("Database exceeds the " + validation.MAX_RECORDS + " record limit (including archived records).", "LIMIT"); + var result = { databaseId: current._id, databaseRevision: current.revision, sourceNamespace: batch.sourceNamespace, createdCount: added.length, replayedCount: batch.rows.length - added.length, mappings: resultMappings }; + // Remember fresh replay operations too, so reusing an operation id with another + // payload still conflicts. Preserve record values, archives and database revision. + if (!added.length) { + ctx.remember(next, operationId, fingerprint, result); + await ctx.replace(current, next); + return Object.assign(result, { projectionPending: current.projectionPending === true, projectionError: current.projectionError || null }); + } + var databases = await ctx.all(); var at = Date.now(); var who = ctx.actor(args.actor); + added.forEach(function (item) { + next.records.push({ id: item.mapping.recordId, revision: 1, values: {}, archivedAt: null, createdAt: at, updatedAt: at, createdBy: who, updatedBy: who }); + next.recordHistory[item.mapping.recordId] = []; + }); + var localDatabases = databases.filter(function (database) { return database._id !== next._id; }).concat([next]); + var context = ctx.referenceContext(localDatabases, next); + added.forEach(function (item, position) { + var values = convert(item.row.values, current.schema, databases, next, planned); + next.records[current.records.length + position].values = validation.normalizeValues(values, current.schema, context, null); + }); + next.importMappings = mappings(next).concat(added.map(function (item) { return item.mapping; })); + next.revision++; next.updatedAt = at; next.updatedBy = who; next.projectionPending = true; next.projectionError = null; + result.databaseRevision = next.revision; + ctx.remember(next, operationId, fingerprint, result); + await ctx.replace(current, next); + return ctx.finish(next, result, who); + }); }); + } + function readImportProgress(args) { + var namespace = input.sourceText(args.sourceNamespace, "Source namespace", 120); + var offset = args.offset === undefined ? 0 : args.offset; var limit = args.limit === undefined ? 100 : args.limit; + if (!Number.isInteger(offset) || offset < 0 || !Number.isInteger(limit) || limit < 1 || limit > 100) throw ctx.error("Import progress requires a nonnegative integer offset and limit between 1 and 100.", "INVALID"); + return ctx.load(args.databaseId).then(function (database) { + var rows = mappings(database).filter(function (mapping) { return mapping.sourceNamespace === namespace; }); + rows.sort(function (a, b) { return a.sourceKey === b.sourceKey ? 0 : a.sourceKey < b.sourceKey ? -1 : 1; }); + return { databaseId: database._id, databaseRevision: database.revision, sourceNamespace: namespace, total: rows.length, mappings: ctx.clone(rows.slice(offset, offset + limit)), nextOffset: offset + limit < rows.length ? offset + limit : null }; + }); + } + return { importRecords: importRecords, readImportProgress: readImportProgress }; +} + +module.exports = { attachKnowledgeDbImport: attachKnowledgeDbImport, MAX_BATCH_ROWS: input.MAX_BATCH_ROWS, MAX_BATCH_BYTES: input.MAX_BATCH_BYTES }; diff --git a/lib/knowledge-db-service.js b/lib/knowledge-db-service.js index 27f0d025..e390dbe3 100644 --- a/lib/knowledge-db-service.js +++ b/lib/knowledge-db-service.js @@ -1,17 +1,17 @@ var crypto = require("crypto"); -var fs = require("fs"); -var path = require("path"); -var Datastore = require("@seald-io/nedb"); var validation = require("./knowledge-db-validation"); +var imports = require("./knowledge-db-import"); +var storageModule = require("./knowledge-db-storage"); var MAX_HISTORY = 50; var MAX_RECEIPTS = 500; var MAX_PAGE = 100; var MAX_FILTERS = 20; var MAX_SORTS = 10; -var MAX_AGGREGATE_BYTES = 8 * 1024 * 1024; +// Synthetic imports use 8.36 MB for 8563 narrative rows and exceed 8 MiB at +// 11400 short rows. Keep finite headroom for the 20000-row envelope and history. +var MAX_AGGREGATE_BYTES = 16 * 1024 * 1024; var IDEMPOTENCY_WINDOW_MS = 24 * 60 * 60 * 1000; -var STORE_REGISTRY = {}; function clone(value) { return JSON.parse(JSON.stringify(value)); } function error(message, code, currentRevision) { return Object.assign(new Error(message), { code: code || "FAILED", currentRevision: currentRevision }); } @@ -107,33 +107,20 @@ function queryRecords(database, args, databases, settings) { } function createKnowledgeDbService(options) { - var settings = options || {}; var projectDir = path.resolve(settings.projectDir); var projectReal = null; var knowledgeDir = path.join(projectDir, "knowledge"); var dataDir = path.join(knowledgeDir, ".clay-db"); var filename = path.join(dataDir, "databases.db"); + var settings = options || {}; var storage = storageModule.createKnowledgeDbStorage(settings); var aggregateLimit = Number.isInteger(settings.maxAggregateBytes) && settings.maxAggregateBytes > 1024 ? Math.min(MAX_AGGREGATE_BYTES, settings.maxAggregateBytes) : MAX_AGGREGATE_BYTES; - var entry = STORE_REGISTRY[filename]; - if (!entry) { entry = { queues: {}, database: null, loading: null, ready: false }; STORE_REGISTRY[filename] = entry; } - function safeDirectory(target, create) { - var stat; try { stat = fs.lstatSync(target); } catch (cause) { if (!create || cause.code !== "ENOENT") throw cause; fs.mkdirSync(target, { recursive: true, mode: 0o700 }); stat = fs.lstatSync(target); } - if (!stat.isDirectory() || stat.isSymbolicLink()) throw error("Knowledge database path is unsafe.", "UNSAFE_PATH"); - var real = fs.realpathSync(target); if (real !== projectReal && !real.startsWith(projectReal + path.sep)) throw error("Knowledge database path escapes the Mate project.", "UNSAFE_PATH"); - } - function open() { - if (entry.loading) return entry.loading; - entry.loading = Promise.resolve().then(function () { projectReal = fs.realpathSync(projectDir); safeDirectory(knowledgeDir, true); safeDirectory(dataDir, true); var stat; try { stat = fs.lstatSync(filename); } catch (cause) { if (cause.code !== "ENOENT") throw cause; } - if (stat && (!stat.isFile() || stat.isSymbolicLink())) throw error("Knowledge database file is unsafe.", "UNSAFE_PATH"); entry.database = new Datastore({ filename: filename }); return entry.database.loadDatabaseAsync(); }).then(function () { return entry.database.ensureIndexAsync({ fieldName: "_id", unique: true }); }).then(function () { entry.ready = true; return entry.database; }).catch(function (cause) { entry.database = null; entry.loading = null; entry.ready = false; throw cause; }); - return entry.loading; - } - function serialize(key, work) { var previous = entry.queues[key] || Promise.resolve(); var next = previous.catch(function () {}).then(work); entry.queues[key] = next.catch(function () {}); return next; } - function load(id) { return open().then(function (store) { return store.findOneAsync({ _id: validation.id(id, "database id") }); }).then(function (found) { if (!found) throw error("Knowledge database not found.", "NOT_FOUND"); return found; }); } - function all() { return open().then(function (store) { return store.findAsync({}); }); } - function listCached(args) { var items = entry.database ? entry.database.getAllData() : []; return { databases: items.filter(function (item) { return args && args.archived === true ? !!item.archivedAt : !item.archivedAt; }).sort(function (a, b) { return b.updatedAt - a.updatedAt || (a._id === b._id ? 0 : a._id < b._id ? -1 : 1); }).map(function (item) { return publicDatabase(item, false); }) }; } - function assertAggregate(next) { if (Buffer.byteLength(JSON.stringify(next), "utf8") > aggregateLimit) throw error("Knowledge database reached its storage limit.", "LIMIT"); } - function replace(current, next) { assertAggregate(next); return entry.database.updateAsync({ _id: current._id, revision: current.revision }, next, {}).then(function (result) { if (result.numAffected !== 1) throw error("This database changed during the operation.", "STALE", current.revision); return next; }); } + function serialize(key, work) { return storage.serialize(key, work); } + function load(id) { return storage.get(validation.id(id, "database id")).then(function (found) { if (!found) throw error("Knowledge database not found.", "NOT_FOUND"); return found; }); } + function all() { return storage.list(); } + function listCached(args) { var items = storage.getCached(); return { databases: items.filter(function (item) { return args && args.archived === true ? !!item.archivedAt : !item.archivedAt; }).sort(function (a, b) { return b.updatedAt - a.updatedAt || (a._id === b._id ? 0 : a._id < b._id ? -1 : 1); }).map(function (item) { return publicDatabase(item, false); }) }; } + function assertAggregate(next) { if (Buffer.byteLength(JSON.stringify(next), "utf8") > aggregateLimit) throw error("Knowledge database exceeds the " + aggregateLimit + " byte aggregate storage limit (records, history, mappings and live receipts).", "LIMIT"); } + function replace(current, next) { assertAggregate(next); return storage.replaceIfRevision(current._id, current.revision, next).then(function (replaced) { if (typeof replaced !== "boolean") throw storageModule.invalid("replaceIfRevision must resolve to a boolean."); if (!replaced) throw error("This database changed during the operation.", "STALE", current.revision); return next; }); } function projectionContent(value) { return JSON.stringify({ schemaVersion: 1, namespace: "knowledge-database-summary", databaseId: value._id, name: value.name, archived: !!value.archivedAt, revision: value.revision, fields: value.schema.filter(function (field) { return !field.archived; }).map(function (field) { return { id: field.id, name: field.name, type: field.type }; }), views: (value.views || []).filter(function (view) { return !view.archived; }).map(function (view) { return { id: view.id, name: view.name, type: "table" }; }), recordCount: value.records.filter(function (record) { return !record.archivedAt; }).length }); } function project(value, actorValue) { if (typeof settings.projectSummary !== "function") return Promise.resolve({ pending: false }); return Promise.resolve().then(function () { return settings.projectSummary({ databaseId: value._id, name: value.name, revision: value.revision, archived: !!value.archivedAt, content: projectionContent(value), actor: actorValue }); }).then(function () { - var clean = clone(value); clean.projectionPending = false; clean.projectionError = null; return entry.database.updateAsync({ _id: value._id, revision: value.revision }, clean, {}).then(function (result) { if (result.numAffected !== 1) return { pending: true, error: "A newer database revision still needs summary projection." }; return { pending: false }; }); - }).catch(function (cause) { var message = cause.message || "Knowledge database summary projection failed."; var failed = clone(value); failed.projectionPending = true; failed.projectionError = message; return entry.database.updateAsync({ _id: value._id, revision: value.revision }, failed, {}).then(function () { return { pending: true, error: message }; }, function () { return { pending: true, error: message }; }); }); + var clean = clone(value); clean.projectionPending = false; clean.projectionError = null; return replace(value, clean).then(function () { return { pending: false }; }); + }).catch(function (cause) { var message = cause.message || "Knowledge database summary projection failed."; var failed = clone(value); failed.projectionPending = true; failed.projectionError = message; return replace(value, failed).then(function () { return { pending: true, error: message }; }, function () { return { pending: true, error: message }; }); }); } function finish(value, result, actorValue) { return project(value, actorValue).then(function (projection) { return Object.assign({}, result, { projectionPending: projection.pending, projectionError: projection.error || null }); }); } function mutate(databaseId, args, label, fn) { @@ -149,7 +136,7 @@ function createKnowledgeDbService(options) { var at = now(); var who = actor(input.actor); var titleId = newId("fld"); var schema = input.schema ? validation.normalizeSchema(input.schema) : validation.normalizeSchema([{ id: titleId, name: "Name", type: "title", required: true }]); var viewId = newId("view"); var item = { _id: newId("db"), name: validation.text(input.name, "Database name", 120, false), revision: 1, archivedAt: null, schema: schema, views: [{ id: viewId, name: "Table", type: "table", columns: schema.filter(function (field) { return !field.archived; }).map(function (field) { return field.id; }), filters: [], sorts: [], archived: false }], records: [], recordHistory: {}, databaseHistory: [], receipts: [], createdAt: at, updatedAt: at, createdBy: who, updatedBy: who, projectionPending: true, projectionError: null }; - var result = { database: publicDatabase(item, false) }; remember(item, operationId, operationFingerprint, result); assertAggregate(item); return open().then(function (store) { return store.insertAsync(item); }).then(function () { return finish(item, result, who); }); + var result = { database: publicDatabase(item, false) }; remember(item, operationId, operationFingerprint, result); assertAggregate(item); return storage.insert(item).then(function () { return finish(item, result, who); }); }); }); } function list(args) { return all().then(function (items) { return { databases: items.filter(function (item) { return args && args.archived === true ? !!item.archivedAt : !item.archivedAt; }).sort(function (a, b) { return b.updatedAt - a.updatedAt || (a._id === b._id ? 0 : a._id < b._id ? -1 : 1); }).map(function (item) { return publicDatabase(item, false); }) }; }); } @@ -163,13 +150,14 @@ function createKnowledgeDbService(options) { function updateSchema(args) { return mutate(args.databaseId, args, "schema", function (next, current) { ensureRevision(args.expectedRevision, current.revision, "database"); var schema = validation.normalizeSchema(args.schema); validation.assertCompatibleSchema(current.schema, schema, current.records); next.databaseHistory = next.databaseHistory.concat([{ revision: current.revision, at: now(), actor: actor(args.actor), action: "schema", schema: current.schema }]).slice(-MAX_HISTORY); next.schema = schema; next.views = reconcileViews(next.views, schema); next.revision++; next.updatedAt = now(); next.updatedBy = actor(args.actor); return { database: publicDatabase(next, false) }; }); } function saveView(args) { return mutate(args.databaseId, args, "view", function (next, current) { ensureRevision(args.expectedRevision, current.revision, "database"); var view = validation.normalizeView(args.view, current.schema); var index = next.views.findIndex(function (entry) { return entry.id === view.id; }); if (index === -1 && next.views.length >= validation.MAX_VIEWS) throw error("Database has too many views.", "LIMIT"); if (index === -1) next.views.push(view); else next.views[index] = view; next.revision++; next.updatedAt = now(); next.updatedBy = actor(args.actor); return { database: publicDatabase(next, false), view: clone(view) }; }); } function referenceContext(databases, current) { return { authorizeReference: function (reference) { if (reference.type === "text") return typeof settings.authorizeTextReference === "function" && settings.authorizeTextReference(reference.id) === true; var target = databases.filter(function (item) { return item._id === reference.databaseId; })[0]; return !!(target && target.records.some(function (record) { return record.id === reference.id; })); } }; } - function createRecord(args) { return all().then(function (databases) { return mutate(args.databaseId, args, "record", function (next, current) { ensureRevision(args.expectedDatabaseRevision, current.revision, "database"); if (next.records.length >= validation.MAX_RECORDS) throw error("Database reached the record limit.", "LIMIT"); var at = now(); var who = actor(args.actor); var record = { id: newId("rec"), revision: 1, values: validation.normalizeValues(args.values || {}, current.schema, referenceContext(databases, current), null), archivedAt: null, createdAt: at, updatedAt: at, createdBy: who, updatedBy: who }; next.records.push(record); next.recordHistory[record.id] = []; next.revision++; next.updatedAt = at; next.updatedBy = who; return { databaseRevision: next.revision, record: publicRecord(record) }; }); }); } + function createRecord(args) { return all().then(function (databases) { return mutate(args.databaseId, args, "record", function (next, current) { ensureRevision(args.expectedDatabaseRevision, current.revision, "database"); if (next.records.length >= validation.MAX_RECORDS) throw error("Database exceeds the " + validation.MAX_RECORDS + " record limit (including archived records).", "LIMIT"); var at = now(); var who = actor(args.actor); var record = { id: newId("rec"), revision: 1, values: validation.normalizeValues(args.values || {}, current.schema, referenceContext(databases, current), null), archivedAt: null, createdAt: at, updatedAt: at, createdBy: who, updatedBy: who }; next.records.push(record); next.recordHistory[record.id] = []; next.revision++; next.updatedAt = at; next.updatedBy = who; return { databaseRevision: next.revision, record: publicRecord(record) }; }); }); } function updateRecord(args) { return all().then(function (databases) { return mutate(args.databaseId, args, "record", function (next, current) { var index = next.records.findIndex(function (record) { return record.id === validation.id(args.recordId, "record id"); }); if (index === -1) throw error("Knowledge database record not found.", "NOT_FOUND"); var before = next.records[index]; ensureRevision(args.expectedRevision, before.revision, "record"); if (before.archivedAt) throw error("Knowledge database record is archived.", "ARCHIVED", before.revision); next.recordHistory[before.id] = (next.recordHistory[before.id] || []).concat([{ revision: before.revision, at: now(), actor: actor(args.actor), action: "update", values: before.values }]).slice(-MAX_HISTORY); var record = clone(before); record.values = validation.normalizeValues(args.values || {}, current.schema, referenceContext(databases, current), record.values); record.revision++; record.updatedAt = now(); record.updatedBy = actor(args.actor); next.records[index] = record; next.revision++; next.updatedAt = record.updatedAt; next.updatedBy = record.updatedBy; return { databaseRevision: next.revision, record: publicRecord(record) }; }); }); } function archiveRecord(args) { return mutate(args.databaseId, args, "record", function (next) { var index = next.records.findIndex(function (record) { return record.id === validation.id(args.recordId, "record id"); }); if (index === -1) throw error("Knowledge database record not found.", "NOT_FOUND"); var before = next.records[index]; ensureRevision(args.expectedRevision, before.revision, "record"); next.recordHistory[before.id] = (next.recordHistory[before.id] || []).concat([{ revision: before.revision, at: now(), actor: actor(args.actor), action: args.recover === true ? "recover" : "archive", values: before.values, archivedAt: before.archivedAt }]).slice(-MAX_HISTORY); var record = clone(before); record.archivedAt = args.recover === true ? null : now(); record.revision++; record.updatedAt = now(); record.updatedBy = actor(args.actor); next.records[index] = record; next.revision++; next.updatedAt = record.updatedAt; next.updatedBy = record.updatedBy; return { databaseRevision: next.revision, record: publicRecord(record) }; }); } function readRecord(args) { return all().then(function (databases) { var item = databases.filter(function (entry) { return entry._id === validation.id(args.databaseId, "database id"); })[0]; if (!item) throw error("Knowledge database not found.", "NOT_FOUND"); var record = item.records.filter(function (entry) { return entry.id === validation.id(args.recordId, "record id"); })[0]; if (!record) throw error("Knowledge database record not found.", "NOT_FOUND"); return { database: publicDatabase(item, false), record: publicRecord(record, referenceStates(item, record, databases, settings)), title: titleValue(item, record), history: clone(item.recordHistory[record.id] || []) }; }); } function query(args) { return all().then(function (databases) { var item = databases.filter(function (entry) { return entry._id === validation.id(args.databaseId, "database id"); })[0]; if (!item) throw error("Knowledge database not found.", "NOT_FOUND"); if (item.archivedAt) throw error("Knowledge database is archived.", "ARCHIVED", item.revision); return queryRecords(item, args, databases, settings); }); } function retryProjection(args) { return serialize(args.databaseId, function () { return load(args.databaseId).then(function (item) { return finish(item, { database: publicDatabase(item, false) }, actor(args.actor)); }); }); } - return { filename: filename, hasStore: function () { try { return fs.lstatSync(filename).isFile(); } catch (cause) { return false; } }, isReady: function () { return entry.ready === true; }, initialize: open, listCached: listCached, create: create, list: list, read: read, rename: rename, archive: archive, recover: function (args) { return archive(Object.assign({}, args, { recover: true })); }, updateSchema: updateSchema, saveView: saveView, createRecord: createRecord, updateRecord: updateRecord, archiveRecord: archiveRecord, recoverRecord: function (args) { return archiveRecord(Object.assign({}, args, { recover: true })); }, readRecord: readRecord, query: query, retryProjection: retryProjection }; + var importer = imports.attachKnowledgeDbImport({ clone: clone, error: error, actor: actor, newId: newId, mutationId: mutationId, fingerprint: fingerprint, ensureRevision: ensureRevision, receipt: receipt, remember: remember, serialize: serialize, load: load, all: all, referenceContext: referenceContext, replace: replace, finish: finish }); + return { readStorageStatus: function () { return typeof storage.diagnostics === "function" ? storage.diagnostics() : Promise.resolve(null); }, importRecords: importer.importRecords, readImportProgress: importer.readImportProgress, hasStore: function () { return storage.hasStore(); }, isReady: function () { return storage.isReady(); }, initialize: function () { return Promise.resolve(storage.initialize()).then(function () {}); }, listCached: listCached, create: create, list: list, read: read, rename: rename, archive: archive, recover: function (args) { return archive(Object.assign({}, args, { recover: true })); }, updateSchema: updateSchema, saveView: saveView, createRecord: createRecord, updateRecord: updateRecord, archiveRecord: archiveRecord, recoverRecord: function (args) { return archiveRecord(Object.assign({}, args, { recover: true })); }, readRecord: readRecord, query: query, retryProjection: retryProjection }; } module.exports = { createKnowledgeDbService: createKnowledgeDbService, MAX_HISTORY: MAX_HISTORY, MAX_PAGE: MAX_PAGE, MAX_AGGREGATE_BYTES: MAX_AGGREGATE_BYTES, IDEMPOTENCY_WINDOW_MS: IDEMPOTENCY_WINDOW_MS }; diff --git a/lib/knowledge-db-storage-nedb-maintenance.js b/lib/knowledge-db-storage-nedb-maintenance.js new file mode 100644 index 00000000..97f37a3a --- /dev/null +++ b/lib/knowledge-db-storage-nedb-maintenance.js @@ -0,0 +1,99 @@ +var fs = require("fs"); + +var MIN_COMPACTION_BYTES = 1024 * 1024; +var MAX_WRITE_AMPLIFICATION = 3; +var MAX_WRITES_WITHOUT_COMPACTION = 64; +var FAILURE_HEADROOM_BYTES = 4 * 1024 * 1024; +var FAILURE_AMPLIFICATION = 6; + +// Shared by every service instance for this filename. No interval or background work: +// maintenance runs inside the same I/O gate as writes, reads and failed-write reloads. +function attachKnowledgeDbNedbMaintenance(ctx) { + var entry = ctx.entry; + if (!entry.storage) entry.storage = { + queue: Promise.resolve(), store: null, cached: [], sizes: new Map(), liveBytes: 0, + journalBytes: 0, peakJournalBytes: 0, journalUnknown: false, successfulWrites: 0, + writesSinceCompaction: 0, compactions: 0, failedCompactions: 0, recoveries: 0, + maintenancePending: false, maintenanceError: null + }; + var state = entry.storage; + function enqueue(work) { + var next = state.queue.catch(function () {}).then(work); + state.queue = next.catch(function () {}); return next; + } + function size(value) { return Buffer.byteLength(JSON.stringify(value), "utf8") + 1; } + function refresh(store) { + state.store = store; state.cached = store.getAllData(); state.sizes = new Map(); state.liveBytes = 0; + state.cached.forEach(function (value) { var bytes = size(value); state.sizes.set(value._id, bytes); state.liveBytes += bytes; }); + } + function issue(cause) { state.maintenancePending = true; state.maintenanceError = cause.message || String(cause); } + async function measure() { + try { + state.journalBytes = (await fs.promises.stat(ctx.filename)).size; state.journalUnknown = false; + state.peakJournalBytes = Math.max(state.peakJournalBytes, state.journalBytes); + return true; + } catch (cause) { state.journalUnknown = true; issue(cause); return false; } + } + async function ensure() { + var store = await ctx.open(); + if (state.store !== store) { refresh(store); await measure(); } + return store; + } + async function compact(store) { + try { + // Supported datastore API also participates in NeDB's own executor. + await store.compactDatafileAsync(); + state.compactions++; state.writesSinceCompaction = 0; + if (await measure()) { state.maintenancePending = false; state.maintenanceError = null; } + } catch (cause) { state.failedCompactions++; issue(cause); await measure(); } + } + function due() { + return state.journalBytes > Math.max(MIN_COMPACTION_BYTES, MAX_WRITE_AMPLIFICATION * state.liveBytes) || + state.writesSinceCompaction >= MAX_WRITES_WITHOUT_COMPACTION && state.journalBytes - state.liveBytes >= MIN_COMPACTION_BYTES; + } + function blocked() { + return state.maintenancePending && (state.journalUnknown || state.journalBytes > Math.max(FAILURE_HEADROOM_BYTES, FAILURE_AMPLIFICATION * state.liveBytes)); + } + function read(work) { return enqueue(async function () { return work(await ensure()); }); } + function write(work, value) { + return enqueue(async function () { + var store = await ensure(); var attempted = false; + if (state.maintenancePending) { attempted = true; await compact(store); } + // Repeated maintenance failure cannot allow unbounded journal growth. This + // rejection happens before the next canonical write, never after its success. + if (blocked()) throw Object.assign(new Error("Knowledge database maintenance must recover before more writes: " + state.maintenanceError), { code: "STORAGE_MAINTENANCE_BLOCKED" }); + var result; + try { result = await work(store); } + catch (cause) { + // NeDB changes indexes before appending. Hold the shared gate through reload + // so another database cannot compact or persist an unsaved in-memory state. + try { + await store.loadDatabaseAsync(); refresh(store); state.recoveries++; state.writesSinceCompaction = 0; + if (await measure()) { state.maintenancePending = false; state.maintenanceError = null; } + } catch (recoveryCause) { state.store = null; state.cached = []; issue(recoveryCause); ctx.invalidate(); } + throw cause; + } + if (result && result.numAffected === 0) return result; + var bytes = size(value); state.liveBytes += bytes - (state.sizes.get(value._id) || 0); state.sizes.set(value._id, bytes); + state.cached = store.getAllData(); state.successfulWrites++; state.writesSinceCompaction++; + if (await measure() && !attempted && due()) await compact(store); + // A maintenance error never changes a successful canonical write into a failure. + return result; + }); + } + function status() { + return read(async function () { + await measure(); + return { maintenancePending: state.maintenancePending, maintenanceError: state.maintenanceError, + liveBytes: state.liveBytes, journalBytes: state.journalBytes, peakJournalBytes: state.peakJournalBytes, + successfulWrites: state.successfulWrites, writesSinceCompaction: state.writesSinceCompaction, + compactions: state.compactions, failedCompactions: state.failedCompactions, recoveries: state.recoveries }; + }); + } + return { read: read, write: write, cached: function () { return state.cached; }, readStorageStatus: status }; +} + +module.exports = { attachKnowledgeDbNedbMaintenance: attachKnowledgeDbNedbMaintenance, + MIN_COMPACTION_BYTES: MIN_COMPACTION_BYTES, MAX_WRITE_AMPLIFICATION: MAX_WRITE_AMPLIFICATION, + MAX_WRITES_WITHOUT_COMPACTION: MAX_WRITES_WITHOUT_COMPACTION, + FAILURE_HEADROOM_BYTES: FAILURE_HEADROOM_BYTES, FAILURE_AMPLIFICATION: FAILURE_AMPLIFICATION }; diff --git a/lib/knowledge-db-storage-nedb.js b/lib/knowledge-db-storage-nedb.js new file mode 100644 index 00000000..d87fa370 --- /dev/null +++ b/lib/knowledge-db-storage-nedb.js @@ -0,0 +1,66 @@ +var fs = require("fs"); +var path = require("path"); +var Datastore = require("@seald-io/nedb"); +var maintenance = require("./knowledge-db-storage-nedb-maintenance"); +var STORE_REGISTRY = new Map(); + +function clone(value) { return JSON.parse(JSON.stringify(value)); } +function error(message, code) { return Object.assign(new Error(message), { code: code }); } +function createKnowledgeDbNedbStorage(options) { + var projectDir = path.resolve(options.projectDir); var projectReal = null; + var knowledgeDir = path.join(projectDir, "knowledge"); var dataDir = path.join(knowledgeDir, ".clay-db"); + var filename = path.join(dataDir, "databases.db"); var entry = STORE_REGISTRY.get(filename); + if (!entry) { entry = { queues: new Map(), database: null, loading: null, ready: false }; STORE_REGISTRY.set(filename, entry); } + function safeDirectory(target, create) { + var stat; + try { stat = fs.lstatSync(target); } + catch (cause) { if (!create || cause.code !== "ENOENT") throw cause; fs.mkdirSync(target, { recursive: true, mode: 0o700 }); stat = fs.lstatSync(target); } + if (!stat.isDirectory() || stat.isSymbolicLink()) throw error("Knowledge database path is unsafe.", "UNSAFE_PATH"); + var real = fs.realpathSync(target); + if (real !== projectReal && !real.startsWith(projectReal + path.sep)) throw error("Knowledge database path escapes the Mate project.", "UNSAFE_PATH"); + } + function open() { + if (entry.loading) return entry.loading; + entry.loading = Promise.resolve().then(function () { + projectReal = fs.realpathSync(projectDir); safeDirectory(knowledgeDir, true); safeDirectory(dataDir, true); + var stat; try { stat = fs.lstatSync(filename); } catch (cause) { if (cause.code !== "ENOENT") throw cause; } + if (stat && (!stat.isFile() || stat.isSymbolicLink())) throw error("Knowledge database file is unsafe.", "UNSAFE_PATH"); + entry.database = new Datastore({ filename: filename }); return entry.database.loadDatabaseAsync(); + }).then(function () { return entry.database.ensureIndexAsync({ fieldName: "_id", unique: true }); }) + .then(function () { entry.ready = true; return entry.database; }) + .catch(function (cause) { invalidate(); throw cause; }); + return entry.loading; + } + function invalidate() { entry.database = null; entry.loading = null; entry.ready = false; } + var io = maintenance.attachKnowledgeDbNedbMaintenance({ entry: entry, filename: filename, open: open, invalidate: invalidate }); + function serialize(scope, work) { + var previous = entry.queues.get(scope) || Promise.resolve(); var next = previous.catch(function () {}).then(function () { return work(); }); + entry.queues.set(scope, next.catch(function () {})); return next; + } + function insert(snapshot) { + var value = clone(snapshot); + return io.write(function (store) { return store.insertAsync(value); }, value).then(function () {}, function (cause) { + if (cause.errorType === "uniqueViolated") throw error("Knowledge storage identity already exists.", "ALREADY_EXISTS"); + throw cause; + }); + } + function replaceIfRevision(id, revision, snapshot) { + var value = clone(snapshot); + if (value._id !== id) return Promise.reject(error("Knowledge storage replacement identity must match.", "INVALID_STORAGE_ID")); + return io.write(function (store) { return store.updateAsync({ _id: id, revision: revision }, value, {}); }, value) + .then(function (result) { return result.numAffected === 1; }); + } + return { + contractVersion: 1, + initialize: function () { return io.read(function () {}); }, + hasStore: function () { try { return fs.lstatSync(filename).isFile(); } catch (cause) { return false; } }, + isReady: function () { return entry.ready === true; }, + get: function (id) { return io.read(function (store) { return store.findOneAsync({ _id: id }); }); }, + list: function () { return io.read(function (store) { return store.findAsync({}); }); }, + getCached: function () { return clone(io.cached()); }, + insert: insert, replaceIfRevision: replaceIfRevision, serialize: serialize, + diagnostics: function () { return io.readStorageStatus().then(function (status) { return Object.assign({ engine: "nedb", automaticCompaction: false }, status); }); } + }; +} + +module.exports = { createKnowledgeDbNedbStorage: createKnowledgeDbNedbStorage }; diff --git a/lib/knowledge-db-storage.js b/lib/knowledge-db-storage.js new file mode 100644 index 00000000..27963e19 --- /dev/null +++ b/lib/knowledge-db-storage.js @@ -0,0 +1,72 @@ +// Version 1 persists opaque JSON aggregate snapshots with a stable _id and revision. +// _id is the existing Knowledge identity, not a query expression. Engines may map it +// to their own primary key; aggregate format/limits remain application responsibilities. +// Each adapter is bound to one server-authorized Mate-project store; get/list cannot +// broaden that scope. Facades for the same store must share locks and committed cache. +// +// initialize() -> Promise; hasStore()/isReady() -> boolean. +// get(id) -> Promise; list() -> Promise; +// getCached() -> snapshot[] (committed, detached, no initialization side effect). +// insert(snapshot) -> Promise (reject ALREADY_EXISTS; never overwrite). +// replaceIfRevision(id, revision, snapshot) -> Promise: atomic compare and +// full replacement, false on missing/mismatch without writing. Same-revision writes +// are allowed for receipts/projection status. Identity must remain unchanged. +// serialize(scope, work) -> Promise: same-scope callbacks exclude each other +// across all facades of one physical store. Callback receives no engine objects and +// may await adapter operations; this is not a multi-aggregate transaction or I/O lock. +// Engines separately serialize reads/writes/maintenance/recovery as needed. A failed +// write may have committed durably; restore readable committed state before releasing +// the I/O gate, or invalidate it. Never turn committed success into maintenance failure. +// Domain operation receipts resolve ambiguous retries; the adapter does not own them. +// diagnostics() -> Promise is optional, engine-specific and never a native handle. + +var METHODS = ["initialize", "hasStore", "isReady", "get", "list", "getCached", "insert", "replaceIfRevision", "serialize"]; +function invalid(message) { return Object.assign(new Error("Invalid Knowledge storage adapter: " + message), { code: "INVALID_STORAGE_ADAPTER" }); } +function validateAdapter(adapter) { + if (!adapter || typeof adapter !== "object" || adapter.contractVersion !== 1) throw invalid("contractVersion must be 1."); + METHODS.forEach(function (name) { if (typeof adapter[name] !== "function") throw invalid(name + " must be a function."); }); + if (adapter.diagnostics !== undefined && typeof adapter.diagnostics !== "function") throw invalid("diagnostics must be a function when supplied."); + return adapter; +} +function bindAdapter(adapter) { + function snapshot(value) { + if (!value || Object.getPrototypeOf(value) !== Object.prototype || typeof value._id !== "string" || !Number.isInteger(value.revision)) throw invalid("snapshot must be a plain aggregate with _id and integer revision."); + return value; + } + function snapshots(values) { if (!Array.isArray(values)) throw invalid("list/getCached must return an array."); values.forEach(snapshot); return values; } + function call(name, args, check) { + var result; + try { result = adapter[name].apply(adapter, args); } catch (cause) { return Promise.reject(cause); } + if (!result || typeof result.then !== "function") return Promise.reject(invalid(name + " must return a Promise.")); + return result.then(check); + } + function flag(name) { var value = adapter[name](); if (typeof value !== "boolean") throw invalid(name + " must return a boolean."); return value; } + var bound = { + contractVersion: 1, + initialize: function () { return call("initialize", [], function () {}); }, + hasStore: function () { return flag("hasStore"); }, isReady: function () { return flag("isReady"); }, + get: function (id) { return call("get", [id], function (value) { if (value === null) return null; snapshot(value); if (value._id !== id) throw invalid("get returned a different identity."); return value; }); }, + list: function () { return call("list", [], snapshots); }, getCached: function () { return snapshots(adapter.getCached()); }, + insert: function (value) { return call("insert", [value], function (result) { if (result !== undefined) throw invalid("insert must resolve without a native result."); }); }, + replaceIfRevision: function (id, revision, value) { return call("replaceIfRevision", [id, revision, value], function (result) { if (typeof result !== "boolean") throw invalid("replaceIfRevision must resolve to a boolean."); return result; }); }, + serialize: function (scope, work) { return call("serialize", [scope, function () { return work(); }], function (result) { return result; }); } + }; + if (typeof adapter.diagnostics === "function") bound.diagnostics = function () { + return call("diagnostics", [], function (result) { try { return JSON.parse(JSON.stringify(result)); } catch (cause) { throw invalid("diagnostics must resolve to JSON data."); } }); + }; + return bound; +} +function createDefaultKnowledgeDbStorage(options) { + // Lazy engine loading: injected adapters can run domain code without NeDB installed. + return require("./knowledge-db-storage-nedb").createKnowledgeDbNedbStorage(options); +} +function createKnowledgeDbStorage(options) { + var settings = options || {}; var adapter; + if (settings.storageAdapter !== undefined) adapter = settings.storageAdapter; + else if (settings.storageFactory !== undefined) { + if (typeof settings.storageFactory !== "function") throw invalid("storageFactory must be a synchronous function."); + adapter = settings.storageFactory({ projectDir: settings.projectDir }); + } else adapter = module.exports.createDefaultKnowledgeDbStorage({ projectDir: settings.projectDir }); + return bindAdapter(validateAdapter(adapter)); +} +module.exports = { createKnowledgeDbStorage: createKnowledgeDbStorage, createDefaultKnowledgeDbStorage: createDefaultKnowledgeDbStorage, validateAdapter: validateAdapter, invalid: invalid }; diff --git a/lib/knowledge-db-validation.js b/lib/knowledge-db-validation.js index 04dd1bba..3f070ad7 100644 Binary files a/lib/knowledge-db-validation.js and b/lib/knowledge-db-validation.js differ diff --git a/lib/mate-knowledge-mate-bound.js b/lib/mate-knowledge-mate-bound.js new file mode 100644 index 00000000..c764e184 --- /dev/null +++ b/lib/mate-knowledge-mate-bound.js @@ -0,0 +1,53 @@ +// Ordinary Mate Knowledge bindings. The service supplies live validation and +// the already-authorized scope; this module only exposes the Mate operations. + +function createMateBound(options) { + var settings = options || {}; + var revalidate = settings.revalidate; + var initial = settings.initial; + + function ownScope() { + return [revalidate().own]; + } + function db(method, args) { + var current = revalidate(); + if (!current.databaseService || typeof current.databaseService[method] !== "function") throw new Error("Knowledge databases are unavailable for this Mate."); + return current.databaseService[method](Object.assign({}, args || {}, { actor: current.actor })); + } + function text(method, args) { + var current = revalidate(); + return settings.textTools[method](current.own, args || {}, current.actor); + } + return { + isClay: false, + mateId: initial.own.mateId, + scopeId: initial.own.scopeId, + listKnowledge: function (args) { return settings.listFor(ownScope(), args || {}, false); }, + searchKnowledge: function (args) { return settings.searchFor(ownScope(), args || {}, false); }, + readKnowledge: function (args) { return settings.readFor(ownScope(), args || {}, false); }, + readKnowledgeDocument: function (args) { return text("read", args); }, + createKnowledgeDocument: function (args) { return text("create", args); }, + updateKnowledgeDocument: function (args) { return text("update", args); }, + listKnowledgeDocumentHistory: function (args) { return text("history", args); }, + restoreKnowledgeDocument: function (args) { return text("restore", args); }, + listDatabases: function (args) { return db("list", args); }, + createDatabase: function (args) { return db("create", args); }, + readDatabase: function (args) { return db("read", args); }, + renameDatabase: function (args) { return db("rename", args); }, + archiveDatabase: function (args) { return db("archive", args); }, + recoverDatabase: function (args) { return db("recover", args); }, + updateDatabaseSchema: function (args) { return db("updateSchema", args); }, + saveTableView: function (args) { return db("saveView", args); }, + queryDatabase: function (args) { return db("query", args); }, + createDatabaseRecord: function (args) { return db("createRecord", args); }, + readDatabaseRecord: function (args) { return db("readRecord", args); }, + updateDatabaseRecord: function (args) { return db("updateRecord", args); }, + archiveDatabaseRecord: function (args) { return db("archiveRecord", args); }, + recoverDatabaseRecord: function (args) { return db("recoverRecord", args); }, + retryDatabaseSync: function (args) { return db("retryProjection", args); }, + importDatabaseRecords: function (args) { return db("importRecords", args); }, + readDatabaseImportProgress: function (args) { return db("readImportProgress", args); }, + }; +} + +module.exports = { createMateBound: createMateBound }; diff --git a/lib/mate-knowledge-mcp-server.js b/lib/mate-knowledge-mcp-server.js index 987d2ae2..753a37a4 100644 --- a/lib/mate-knowledge-mcp-server.js +++ b/lib/mate-knowledge-mcp-server.js @@ -9,11 +9,17 @@ var buildShape = require("./session-spawn-mcp-server").buildShape; var service = require("./mate-knowledge-service"); +var zod; +try { zod = require("zod"); } catch (error) { zod = null; } var MATE_CONTRACT = "Your Knowledge is the durable personal and expertise context you own: what you have learned, the material you keep, and your accumulated observations. " + "It is yours alone. There are no other Knowledge collections available to you here, and project Logs are a separate, unrelated surface. " + + "Use text documents for narrative or multiline Markdown content, and databases for structured records, relationships, filters, and numeric totals. " + "Search it when recalling something specific would genuinely change your answer, and read only the records you actually need. " + + "Database list and search results are summaries for database discovery, not database row content: discover a database, read its schema, then query_database with stable field ids, typed filters, explicit sorts, bounded pagination, and totals; read individual rows by stable record id. Text search and text-document reads retain their own content and revision semantics. " + + "For text edits, use the returned document id and revision, reread after a conflict, and retry with the current revision. " + + "Treat errors as failures. A successful revision/id with indexSyncPending or projectionPending means canonical content was saved but indexing is pending: report that exact state, do not claim it is indexed, and do not blindly retry a mutation that could create a duplicate. " + "Do not enumerate or restate your Knowledge to the user unasked, and never reproduce whole records as filler."; var CLAY_CONTRACT = @@ -56,24 +62,35 @@ var OPERATION = { type: "string", description: "Stable retry id formatted label_ var FIELD = { type: "object", description: "Field with stable id, name, type, required, and type-specific options or reference target." }; var FILTER = { type: "object", description: "Validated filter: fieldId, op, and value. Supported ops are eq, neq, contains, gt, gte, lt, lte, empty, not-empty." }; var SORT = { type: "object", description: "Validated sort with fieldId and asc or desc direction." }; +var FIELD_SCHEMA = zod ? zod.object({ id: zod.string(), name: zod.string(), type: zod.string(), required: zod.boolean().optional(), options: zod.any().optional(), target: zod.string().optional(), targetDatabaseId: zod.string().optional(), archived: zod.boolean().optional() }) : null; +var FILTER_SCHEMA = zod ? zod.object({ fieldId: zod.string(), op: zod.string(), value: zod.any().optional() }) : null; +var SORT_SCHEMA = zod ? zod.object({ fieldId: zod.string(), direction: zod.enum(["asc", "desc"]) }) : null; +var VALUES_SCHEMA = zod ? zod.record(zod.string(), zod.any()) : null; +var VIEW_SCHEMA = zod ? zod.object({ id: zod.string(), name: zod.string(), columns: zod.array(zod.string()), filters: zod.array(FILTER_SCHEMA || zod.any()).optional(), sorts: zod.array(SORT_SCHEMA || zod.any()).optional(), archived: zod.boolean().optional() }) : null; +var IMPORT_ROW_SCHEMA = zod ? zod.object({ sourceKey: zod.string(), sourcePayload: zod.any(), values: zod.record(zod.string(), zod.any()) }) : null; +var IMPORT_ROW = IMPORT_ROW_SCHEMA ? { schema: IMPORT_ROW_SCHEMA, description: "Source row with sourceKey, immutable JSON sourcePayload, and typed values keyed by stable schema field id. Record references use sourceNamespace/sourceKey, including forward references within this batch." } : { type: "object", description: "Source row with sourceKey, immutable JSON sourcePayload, and typed values keyed by stable schema field id. Record references use sourceNamespace/sourceKey, including forward references within this batch." }; +var DOCUMENT_ID = { type: "string", description: "Stable document id returned by a text-document tool." }; +var DOCUMENT_REVISION = { type: "string", description: "Revision returned by the most recent text-document read or mutation; required for updates and restore." }; function databaseTools(bound) { return [ - { name: "list_databases", description: "List canonical Knowledge databases, including archived databases when requested, so the Mate can discover and recover them without relying on summary projection.", inputSchema: buildShape({ archived: { type: "boolean" } }, []), handler: handler(bound, "listDatabases") }, - { name: "create_database", description: "Create a first-class database in this Mate's Knowledge. Defaults to one required title field and a Table view.", inputSchema: buildShape({ name: { type: "string" }, schema: { type: "array", items: FIELD }, operationId: OPERATION }, ["name", "operationId"]), handler: handler(bound, "createDatabase") }, - { name: "read_database", description: "Read one Knowledge database's current schema, views, revision, and record count.", inputSchema: buildShape({ databaseId: ID, includeArchived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "readDatabase") }, + { name: "list_databases", description: "Discover canonical Knowledge databases, including archived databases when requested. Results are database summaries only, not row content; read a chosen database before querying it.", inputSchema: buildShape({ archived: { type: "boolean" } }, []), handler: handler(bound, "listDatabases") }, + { name: "create_database", description: "Create a first-class structured database in this Mate's Knowledge. Define typed fields for values, relationships, and numeric totals; use text documents for narrative or multiline material.", inputSchema: buildShape({ name: { type: "string" }, schema: FIELD_SCHEMA ? { schema: zod.array(FIELD_SCHEMA), description: "Typed field descriptors." } : { type: "array", items: FIELD }, operationId: OPERATION }, ["name", "operationId"]), handler: handler(bound, "createDatabase") }, + { name: "read_database", description: "Read one discovered Knowledge database by stable database id to obtain its current schema, views, revision, and record count before querying rows.", inputSchema: buildShape({ databaseId: ID, includeArchived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "readDatabase") }, { name: "rename_database", description: "Rename a Knowledge database with optimistic revision checking.", inputSchema: buildShape({ databaseId: ID, name: { type: "string" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "name", "expectedRevision", "operationId"]), handler: handler(bound, "renameDatabase") }, { name: "archive_database", description: "Recoverably archive a Knowledge database.", inputSchema: buildShape({ databaseId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "expectedRevision", "operationId"]), handler: handler(bound, "archiveDatabase") }, { name: "recover_database", description: "Recover an archived Knowledge database.", inputSchema: buildShape({ databaseId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "expectedRevision", "operationId"]), handler: handler(bound, "recoverDatabase") }, - { name: "update_database_schema", description: "Replace the schema using stable field ids. Exactly one active title field is required; populated fields cannot be removed or change type and should be archived instead.", inputSchema: buildShape({ databaseId: ID, schema: { type: "array", items: FIELD }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "schema", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseSchema") }, - { name: "save_table_view", description: "Create or revise a persisted Table view with selected columns, validated filters, and deterministic sorts.", inputSchema: buildShape({ databaseId: ID, view: { type: "object", description: "View with stable id, name, columns, filters, and sorts." }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "view", "expectedRevision", "operationId"]), handler: handler(bound, "saveTableView") }, - { name: "query_database", description: "Query a bounded deterministic page of records using a saved view or explicit validated filters and sorts. Returns count and numeric sums.", inputSchema: buildShape({ databaseId: ID, viewId: ID, filters: { type: "array", items: FILTER }, sorts: { type: "array", items: SORT }, cursor: { type: "string" }, limit: { type: "number" }, archived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "queryDatabase") }, - { name: "create_database_record", description: "Create a typed record. Values are keyed by stable field id; unknown fields and unsafe keys are rejected.", inputSchema: buildShape({ databaseId: ID, values: { type: "object" }, expectedDatabaseRevision: REVISION, operationId: OPERATION }, ["databaseId", "values", "expectedDatabaseRevision", "operationId"]), handler: handler(bound, "createDatabaseRecord") }, + { name: "update_database_schema", description: "Replace the schema using stable field ids. Exactly one active title field is required; populated fields cannot be removed or change type and should be archived instead.", inputSchema: buildShape({ databaseId: ID, schema: FIELD_SCHEMA ? { schema: zod.array(FIELD_SCHEMA), description: "Complete typed field descriptor array." } : { type: "array", items: FIELD }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "schema", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseSchema") }, + { name: "save_table_view", description: "Create or revise a persisted Table view with selected columns, validated filters, and deterministic sorts.", inputSchema: buildShape({ databaseId: ID, view: VIEW_SCHEMA ? { schema: VIEW_SCHEMA, description: "View with stable id, name, columns, filters, and sorts." } : { type: "object", description: "View with stable id, name, columns, filters, and sorts." }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "view", "expectedRevision", "operationId"]), handler: handler(bound, "saveTableView") }, + { name: "query_database", description: "Query a bounded deterministic page using stable field ids, typed validated filters, explicit sorts, and a cursor. Returns only the requested rows plus count and numeric sums for the same filter scope; do not infer totals from one page.", inputSchema: buildShape({ databaseId: ID, viewId: ID, filters: FILTER_SCHEMA ? { schema: zod.array(FILTER_SCHEMA) } : { type: "array", items: FILTER }, sorts: SORT_SCHEMA ? { schema: zod.array(SORT_SCHEMA) } : { type: "array", items: SORT }, cursor: { type: "string" }, limit: { type: "number" }, archived: { type: "boolean" } }, ["databaseId"]), handler: handler(bound, "queryDatabase") }, + { name: "create_database_record", description: "Create a typed record. Values are keyed by stable field id; unknown fields and unsafe keys are rejected.", inputSchema: buildShape({ databaseId: ID, values: VALUES_SCHEMA ? { schema: VALUES_SCHEMA } : { type: "object" }, expectedDatabaseRevision: REVISION, operationId: OPERATION }, ["databaseId", "values", "expectedDatabaseRevision", "operationId"]), handler: handler(bound, "createDatabaseRecord") }, { name: "read_database_record", description: "Read one record with its values, revision, attribution, archive state, and bounded history.", inputSchema: buildShape({ databaseId: ID, recordId: ID }, ["databaseId", "recordId"]), handler: handler(bound, "readDatabaseRecord") }, - { name: "update_database_record", description: "Update selected typed values using the record revision. Edits to other fields made since a prior save are not overwritten.", inputSchema: buildShape({ databaseId: ID, recordId: ID, values: { type: "object" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "values", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseRecord") }, + { name: "update_database_record", description: "Update selected typed values using the record revision. Edits to other fields made since a prior save are not overwritten.", inputSchema: buildShape({ databaseId: ID, recordId: ID, values: VALUES_SCHEMA ? { schema: VALUES_SCHEMA } : { type: "object" }, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "values", "expectedRevision", "operationId"]), handler: handler(bound, "updateDatabaseRecord") }, { name: "archive_database_record", description: "Recoverably archive one record using its current revision.", inputSchema: buildShape({ databaseId: ID, recordId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "expectedRevision", "operationId"]), handler: handler(bound, "archiveDatabaseRecord") }, { name: "recover_database_record", description: "Recover one archived record using its current revision.", inputSchema: buildShape({ databaseId: ID, recordId: ID, expectedRevision: REVISION, operationId: OPERATION }, ["databaseId", "recordId", "expectedRevision", "operationId"]), handler: handler(bound, "recoverDatabaseRecord") }, { name: "retry_database_sync", description: "Retry the shared append-only Knowledge summary projection after a canonical database mutation reported projectionPending.", inputSchema: buildShape({ databaseId: ID }, ["databaseId"]), handler: handler(bound, "retryDatabaseSync") }, + { name: "import_database_records", description: "Import up to 100 source rows (256 KiB per batch) into a discovered database using immutable source identities and typed values. Use sourceNamespace/sourceKey references for same-batch or already-imported cross-database links; replay the exact operation and payload after uncertainty, and reread the current database revision before a stale retry.", inputSchema: buildShape({ databaseId: ID, expectedDatabaseRevision: REVISION, operationId: OPERATION, sourceNamespace: { type: "string", description: "Stable source namespace for this import mapping." }, rows: IMPORT_ROW_SCHEMA ? { schema: zod.array(IMPORT_ROW_SCHEMA), description: "At most 100 source rows and 256 KiB of serialized row JSON." } : { type: "array", description: "At most 100 source rows and 256 KiB of serialized row JSON.", items: IMPORT_ROW } }, ["databaseId", "expectedDatabaseRevision", "operationId", "sourceNamespace", "rows"]), handler: handler(bound, "importDatabaseRecords") }, + { name: "read_database_import_progress", description: "Read bounded source-to-native record mappings by database id and source namespace. Use returned stable record ids for later corrections; this does not read arbitrary filesystem paths or caller-selected owners.", inputSchema: buildShape({ databaseId: ID, sourceNamespace: { type: "string" }, offset: { type: "number" }, limit: { type: "number", description: "Mapping page size from 1 to 100." } }, ["databaseId", "sourceNamespace"]), handler: handler(bound, "readDatabaseImportProgress") }, ].map(function (tool) { tool.description = MATE_CONTRACT + " " + tool.description; return tool; }); } @@ -110,6 +127,36 @@ function mateTools(bound) { }, ["ref"]), handler: handler(bound, "readKnowledge"), }, + { + name: "read_knowledge_document", + description: MATE_CONTRACT + " Read the complete current Markdown document by its relative path or stable document id, including its current revision.", + inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, documentId: DOCUMENT_ID }, []), + handler: handler(bound, "readKnowledgeDocument"), + }, + { + name: "create_knowledge_document", + description: MATE_CONTRACT + " Create a new Markdown document at a safe relative path. Use this for narrative or multiline content; it is synchronized into Knowledge search after the canonical save.", + inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, content: { type: "string", description: "Complete Markdown document content." } }, ["name", "content"]), + handler: handler(bound, "createKnowledgeDocument"), + }, + { + name: "update_knowledge_document", + description: MATE_CONTRACT + " Update an existing Markdown document by relative path or stable document id. The current revision is mandatory, and stale writes are rejected.", + inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, documentId: DOCUMENT_ID, content: { type: "string", description: "Complete replacement Markdown document content." }, expectedRevision: DOCUMENT_REVISION }, ["content", "expectedRevision"]), + handler: handler(bound, "updateKnowledgeDocument"), + }, + { + name: "list_knowledge_document_history", + description: MATE_CONTRACT + " List recoverable revisions for one Markdown document by relative path or stable document id.", + inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, documentId: DOCUMENT_ID }, []), + handler: handler(bound, "listKnowledgeDocumentHistory"), + }, + { + name: "restore_knowledge_document", + description: MATE_CONTRACT + " Restore a selected historical Markdown revision. The current revision is mandatory so a concurrent edit cannot be overwritten.", + inputSchema: buildShape({ name: { type: "string", description: "Relative Markdown path under this Mate's Knowledge directory." }, documentId: DOCUMENT_ID, version: { type: "string", description: "Version returned by list_knowledge_document_history." }, expectedRevision: DOCUMENT_REVISION }, ["version", "expectedRevision"]), + handler: handler(bound, "restoreKnowledgeDocument"), + }, ].concat(databaseTools(bound)); } diff --git a/lib/mate-knowledge-service.js b/lib/mate-knowledge-service.js index b2a8cff9..712fd457 100644 --- a/lib/mate-knowledge-service.js +++ b/lib/mate-knowledge-service.js @@ -15,6 +15,8 @@ var builtinMates = require("./builtin-mates"); var knowledgeImport = require("./knowledge-import"); var knowledgeSearch = require("./knowledge-search"); var mateSync = require("./mate-knowledge-sync"); +var textToolsModule = require("./mate-knowledge-text-tools"); +var mateBoundModule = require("./mate-knowledge-mate-bound"); var MAX_PAGE = 50; var DEFAULT_PAGE = 20; @@ -97,6 +99,7 @@ function attachMateKnowledgeService(ctx) { var listMates = context.listMates || function () { return []; }; var baseDir = context.baseDir; var importerCache = new Map(); + var textTools = textToolsModule.createMateKnowledgeTextTools({ baseDir: baseDir }); function importerForScope(scopeId) { if (context.openImporter) return context.openImporter(scopeId); @@ -379,6 +382,8 @@ function attachMateKnowledgeService(ctx) { if (source.session) { var manager = project.getSessionManager ? project.getSessionManager() : project.sm; if (!manager || !manager.sessions || manager.sessions.get(source.session.localId) !== source.session) return null; + if (source.projectOwnerId && source.session.ownerId && source.session.ownerId !== source.projectOwnerId) return null; + if (isMultiUser() && (!source.projectOwnerId || source.session.ownerId !== source.projectOwnerId)) return null; } if (typeof resolveMate !== "function") return null; var mate = resolveMate(source.projectOwnerId || null, source.mateId); @@ -422,36 +427,10 @@ function attachMateKnowledgeService(ctx) { // An ordinary Mate's own scope, and nothing else. There is no argument on // any tool that could widen it. function createMateBound(revalidate, initial) { - function ownScope() { - return [revalidate().own]; - } - function db(method, args) { - var current = revalidate(); if (!current.databaseService || typeof current.databaseService[method] !== "function") throw new Error("Knowledge databases are unavailable for this Mate."); - return current.databaseService[method](Object.assign({}, args || {}, { actor: current.actor })); - } - return { - isClay: false, - mateId: initial.own.mateId, - scopeId: initial.own.scopeId, - listKnowledge: function (args) { return listFor(ownScope(), args || {}, false); }, - searchKnowledge: function (args) { return searchFor(ownScope(), args || {}, false); }, - readKnowledge: function (args) { return readFor(ownScope(), args || {}, false); }, - listDatabases: function (args) { return db("list", args); }, - createDatabase: function (args) { return db("create", args); }, - readDatabase: function (args) { return db("read", args); }, - renameDatabase: function (args) { return db("rename", args); }, - archiveDatabase: function (args) { return db("archive", args); }, - recoverDatabase: function (args) { return db("recover", args); }, - updateDatabaseSchema: function (args) { return db("updateSchema", args); }, - saveTableView: function (args) { return db("saveView", args); }, - queryDatabase: function (args) { return db("query", args); }, - createDatabaseRecord: function (args) { return db("createRecord", args); }, - readDatabaseRecord: function (args) { return db("readRecord", args); }, - updateDatabaseRecord: function (args) { return db("updateRecord", args); }, - archiveDatabaseRecord: function (args) { return db("archiveRecord", args); }, - recoverDatabaseRecord: function (args) { return db("recoverRecord", args); }, - retryDatabaseSync: function (args) { return db("retryProjection", args); }, - }; + return mateBoundModule.createMateBound({ + revalidate: revalidate, initial: initial, textTools: textTools, + listFor: listFor, searchFor: searchFor, readFor: readFor, + }); } // Every Mate belonging to the same user, resolved fresh on each call so a diff --git a/lib/mate-knowledge-text-tools.js b/lib/mate-knowledge-text-tools.js new file mode 100644 index 00000000..f265682b --- /dev/null +++ b/lib/mate-knowledge-text-tools.js @@ -0,0 +1,106 @@ +// Session-bound text Knowledge mutations for one authorized Mate. +// +// This module deliberately uses the existing portable file store and +// write-through synchronizer. It never accepts a Mate, owner, or filesystem +// root from a tool caller; the caller supplies only a relative document name +// or the stable identity returned by the store. + +var crypto = require("crypto"); +var path = require("path"); +var knowledgeFiles = require("./knowledge-file-store"); +var mateSync = require("./mate-knowledge-sync"); + +function metadataRoot(baseDir, mateDir) { + var key = crypto.createHash("sha256").update(path.resolve(mateDir)).digest("hex").slice(0, 24); + return path.join(baseDir || path.join(mateDir, ".clay"), "knowledge-documents", key); +} + +function failedSync(summary) { + if (!summary) return { indexSyncPending: true, syncError: "Knowledge index synchronization returned no result." }; + if (summary.failed === 0) return { indexSyncPending: false, syncError: null }; + var error = summary.errors && summary.errors[0]; + return { indexSyncPending: true, syncError: error && error.message || "Knowledge index synchronization failed." }; +} + +function createMateKnowledgeTextTools(options) { + var settings = options || {}; + var stores = new Map(); + + function storeFor(mate) { + if (!mate || typeof mate.mateDir !== "string") throw new Error("Knowledge text tools require an authorized Mate."); + var directory = path.resolve(mate.mateDir); + if (!stores.has(directory)) stores.set(directory, knowledgeFiles.createKnowledgeFileStore(directory, { metaDir: metadataRoot(settings.baseDir, directory) })); + return stores.get(directory); + } + + function find(store, input) { + var args = input || {}; + var hasName = typeof args.name === "string" && args.name.length > 0; + var hasId = typeof args.documentId === "string" && args.documentId.length > 0; + if (!hasName && !hasId) throw new Error("A document name or stable document id is required."); + var item = null; + if (hasId) item = store.list().filter(function (entry) { return entry.id === args.documentId; })[0] || null; + if (hasId && !item) throw Object.assign(new Error("Knowledge document not found."), { code: "ENOENT" }); + if (hasName) { + var byName = store.read(args.name); + if (item && item.id !== byName.id) throw new Error("The document name and id identify different documents."); + item = byName; + } + if (!item) throw Object.assign(new Error("Knowledge document not found."), { code: "ENOENT" }); + if (hasId && !hasName) item = store.read(item.name); + if (item.itemType !== "text") throw Object.assign(new Error("Only Markdown Knowledge documents are writable through these tools."), { code: "READ_ONLY" }); + return item; + } + + function sync(mate, name, actor) { + try { + var summary = mateSync.syncMateSource({ mateDir: mate.mateDir, fileName: name, baseDir: settings.baseDir, actor: actor }); + return failedSync(summary); + } catch (error) { + return { indexSyncPending: true, syncError: error.message || "Knowledge index synchronization failed." }; + } + } + + function read(mate, args) { + var item = find(storeFor(mate), args); + return { documentId: item.id, name: item.name, content: item.content, revision: item.revision }; + } + + function create(mate, args, actor) { + var input = args || {}; + if (typeof input.name !== "string" || !input.name) throw new Error("A document name is required to create a Knowledge document."); + if (typeof input.content !== "string") throw new Error("Document content is required."); + var store = storeFor(mate); + var saved = store.save(input.name, input.content, null); + return Object.assign({ documentId: saved.id, name: saved.name, revision: saved.revision }, sync(mate, saved.name, actor)); + } + + function update(mate, args, actor) { + var input = args || {}; + if (typeof input.expectedRevision !== "string" || !input.expectedRevision) throw Object.assign(new Error("A current document revision is required for updates."), { code: "STALE" }); + if (typeof input.content !== "string") throw new Error("Document content is required."); + var store = storeFor(mate); + var current = find(store, input); + var saved = store.save(current.name, input.content, input.expectedRevision); + return Object.assign({ documentId: saved.id, name: saved.name, revision: saved.revision }, sync(mate, saved.name, actor)); + } + + function history(mate, args) { + var current = find(storeFor(mate), args); + return { documentId: current.id, name: current.name, versions: storeFor(mate).history(current.id) }; + } + + function restore(mate, args, actor) { + var input = args || {}; + if (typeof input.expectedRevision !== "string" || !input.expectedRevision) throw Object.assign(new Error("A current document revision is required for restore."), { code: "STALE" }); + if (typeof input.version !== "string" || !input.version) throw new Error("A history version is required."); + var store = storeFor(mate); + var current = find(store, input); + var restored = store.restore(current.id, input.version, current.name, input.expectedRevision); + return Object.assign({ documentId: restored.id, name: restored.name, revision: restored.revision }, sync(mate, restored.name, actor)); + } + + return { read: read, create: create, update: update, history: history, restore: restore }; +} + +module.exports = { createMateKnowledgeTextTools: createMateKnowledgeTextTools }; diff --git a/lib/project-mate-interaction.js b/lib/project-mate-interaction.js index d3c0be5e..2cb61578 100644 --- a/lib/project-mate-interaction.js +++ b/lib/project-mate-interaction.js @@ -3,6 +3,7 @@ var fs = require("fs"); var path = require("path"); var crypto = require("crypto"); var mateSync = require("./mate-knowledge-sync"); +var dmDigestCursor = require("./dm-digest-cursor"); // Mirror a Mate-owned journal into the Clay-wide Knowledge backend after the // legacy append has already succeeded. Reconciling the whole journal keeps the @@ -294,6 +295,9 @@ function attachMateInteraction(ctx) { mirrorMateJournal(mateDir, "session-digests.jsonl", job); } catch (e) { console.error("[digest-worker] Write failed for " + job.mateId + ":", e.message); + if (job.onError) job.onError(e); + processDigestQueue(); + return; } // Write user observations if present @@ -422,63 +426,19 @@ function attachMateInteraction(ctx) { // Delta-based: only collects new turns since the last successful digest. // Concurrency debounce: turns that arrive while a digest is in-flight // are naturally batched into the next flush. - var _dmDigestPending = false; - function digestDmTurn(session, responsePreview) { - if (!isMate || _dmDigestPending) return; + var _dmDigestPending = new WeakSet(); + function digestDmTurn(session) { + if (!isMate || _dmDigestPending.has(session)) return; var mateId = path.basename(cwd); var mateCtx = matesModule.buildMateCtx(projectOwnerId); if (!matesModule.isMate(mateCtx, mateId)) return; - // Track digest index per session so switching sessions doesn't misalign. - // On resumed sessions (after restart), recover index from the last - // digest_checkpoint entry in history so undigested turns aren't lost. - if (typeof session._dmLastDigestedIndex !== "number") { - session._dmLastDigestedIndex = 0; - for (var ci = session.history.length - 1; ci >= 0; ci--) { - if (session.history[ci].type === "digest_checkpoint") { - session._dmLastDigestedIndex = session.history[ci].digestedIndex; - break; - } - } - } - - // Collect only new turns since the last successful digest - var conversationParts = []; - var totalLen = 0; - var CONV_CAP = 6000; - var startIndex = session._dmLastDigestedIndex; - for (var hi = startIndex; hi < session.history.length; hi++) { - if (totalLen >= CONV_CAP) break; - var entry = session.history[hi]; - if (entry.type === "user_message" && entry.text) { - var uText = entry.text; - if (totalLen + uText.length > CONV_CAP) { - uText = uText.substring(0, Math.max(200, CONV_CAP - totalLen)) + "..."; - } - conversationParts.push("User: " + uText); - totalLen += uText.length; - } else if (entry.type === "assistant_message" && entry.text) { - var aText = entry.text; - if (totalLen + aText.length > CONV_CAP) { - aText = aText.substring(0, Math.max(200, CONV_CAP - totalLen)) + "..."; - } - conversationParts.push("Mate: " + aText); - totalLen += aText.length; - } - } - // If the latest response hasn't landed in history yet, append it - var lastResponseText = responsePreview || ""; - if (lastResponseText && conversationParts.length > 0) { - var lastPart = conversationParts[conversationParts.length - 1]; - if (lastPart.indexOf("Mate:") !== 0 || lastPart.indexOf(lastResponseText.substring(0, 50)) === -1) { - var rText = lastResponseText; - if (totalLen + rText.length > CONV_CAP) { - rText = rText.substring(0, Math.max(200, CONV_CAP - totalLen)) + "..."; - } - conversationParts.push("Mate: " + rText); - } - } - if (conversationParts.length === 0) return; + // Recover the exact next unread position. Legacy index-only checkpoints + // remain readable and are treated as fully consumed entries. + var digestState = dmDigestCursor.readState(session.history); + var startCursor = digestState.cursor; + var chunk = dmDigestCursor.collectChunk(session.history, startCursor); + if (!chunk.conversationContent) return; var mateDir = matesModule.getMateDir(mateCtx, mateId); var knowledgeDir = path.join(mateDir, "knowledge"); @@ -500,8 +460,8 @@ function attachMateInteraction(ctx) { } } catch (e) {} - _dmDigestPending = true; - var snapshotIndex = session.history.length; + _dmDigestPending.add(session); + var checkpointCursor = chunk.cursor; enqueueDigest({ mateCtx: mateCtx, @@ -509,17 +469,27 @@ function attachMateInteraction(ctx) { mateId: mateId, type: "dm", priorSummary: priorSummary || "", - conversationContent: conversationParts.join("\n"), + conversationContent: chunk.conversationContent, onDone: function () { - session._dmLastDigestedIndex = snapshotIndex; - // Persist checkpoint so resumed sessions know where to continue - var checkpoint = { type: "digest_checkpoint", digestedIndex: snapshotIndex }; + // Journal and checkpoint are separate appends: if this append fails, + // retry may duplicate the saved digest, but must never skip its input. + var checkpoint = { type: "digest_checkpoint", cursor: checkpointCursor }; + var saved = false; + try { saved = sm.appendToSessionFile(session, checkpoint) !== false; } catch (e) { + console.error("[dm-digest] Checkpoint append failed:", e.message); + } + if (!saved) { + _dmDigestPending.delete(session); + return; + } session.history.push(checkpoint); - sm.appendToSessionFile(session, checkpoint); - _dmDigestPending = false; + _dmDigestPending.delete(session); + if (dmDigestCursor.hasPending(session.history, checkpointCursor)) { + setImmediate(function () { digestDmTurn(session, ""); }); + } }, onError: function () { - _dmDigestPending = false; + _dmDigestPending.delete(session); }, }); } diff --git a/test/dm-digest-cursor.test.js b/test/dm-digest-cursor.test.js new file mode 100644 index 00000000..2967bf19 --- /dev/null +++ b/test/dm-digest-cursor.test.js @@ -0,0 +1,311 @@ +var test = require("node:test"); +var assert = require("node:assert/strict"); +var fs = require("node:fs"); +var os = require("node:os"); +var path = require("node:path"); +var cursor = require("../lib/dm-digest-cursor"); +var attachMateInteraction = require("../lib/project-mate-interaction").attachMateInteraction; + +function user(text) { return { type: "user_message", text: text }; } +function mate(text) { return { type: "assistant_message", text: text }; } + +test("chunks a conversation at the cap and preserves oversized entry progress", function () { + var history = [user("x".repeat(12000))]; + var first = cursor.collectChunk(history, { historyIndex: 0, textOffset: 0 }, ""); + assert.equal(first.conversationContent.length, cursor.CONVERSATION_CAP); + assert.deepEqual(first.cursor, { historyIndex: 0, textOffset: cursor.CONVERSATION_CAP - 6 }); + + var second = cursor.collectChunk(history, first.cursor, ""); + assert.equal(second.conversationContent.length, cursor.CONVERSATION_CAP); + var third = cursor.collectChunk(history, second.cursor, ""); + assert.equal(third.conversationContent, "User: " + "x".repeat(12000 - second.cursor.textOffset)); + assert.deepEqual(third.cursor, { historyIndex: 1, textOffset: 0 }); + assert.equal(cursor.hasPending(history, third.cursor), false); +}); + +test("serialized conversation cap includes labels, separators, and every small entry", function () { + var history = []; + for (var i = 0; i < 1000; i++) history.push(user("a")); + var chunk = cursor.collectChunk(history, { historyIndex: 0, textOffset: 0 }, ""); + assert.ok(chunk.conversationContent.length <= cursor.CONVERSATION_CAP); + assert.equal(chunk.conversationContent, Array(750).join("User: a\n") + "User: a"); + assert.equal(chunk.cursor.historyIndex, 750); +}); + +test("oversized chunks do not split surrogate pairs", function () { + var history = [user("a".repeat(5994) + "😀" + "b")]; + var first = cursor.collectChunk(history, { historyIndex: 0, textOffset: 0 }, ""); + assert.equal(first.conversationContent.charCodeAt(first.conversationContent.length - 1), "a".charCodeAt(0)); + var second = cursor.collectChunk(history, first.cursor, ""); + assert.equal(second.conversationContent, "User: 😀b"); +}); + +test("legacy index checkpoints resume at the next entry after restart", function () { + var history = [user("first"), mate("reply"), { type: "digest_checkpoint", digestedIndex: 2 }, user("second")]; + var start = cursor.readCursor(history); + assert.deepEqual(start, { historyIndex: 2, textOffset: 0 }); + var chunk = cursor.collectChunk(history, start, ""); + assert.match(chunk.conversationContent, /User: second/); +}); + +test("persisted mid-message checkpoint restores the exact cursor after restart", function () { + var root = fs.mkdtempSync(path.join(os.tmpdir(), "clay-dm-cursor-")); + var checkpointFile = path.join(root, "session.jsonl"); + var history = [user("x".repeat(10000))]; + var first = cursor.collectChunk(history, { historyIndex: 0, textOffset: 0 }, ""); + fs.writeFileSync(checkpointFile, JSON.stringify({ type: "user_message", text: history[0].text }) + "\n" + + JSON.stringify({ type: "digest_checkpoint", cursor: first.cursor }) + "\n"); + var restored = fs.readFileSync(checkpointFile, "utf8").trim().split("\n").map(function (line) { return JSON.parse(line); }); + assert.deepEqual(cursor.readCursor(restored), first.cursor); + assert.equal(cursor.collectChunk(restored, cursor.readCursor(restored), "").conversationContent, + "User: " + "x".repeat(10000 - first.cursor.textOffset)); + fs.rmSync(root, { recursive: true, force: true }); +}); + +test("fresh preview never consumes an unread historical assistant entry", function () { + var history = [user("question"), mate("the answer is ready")]; + var chunk = cursor.collectChunk(history, { historyIndex: 0, textOffset: 0 }, "the answer is ready"); + assert.equal(chunk.conversationContent, "User: question\nMate: the answer is ready"); + assert.deepEqual(chunk.cursor, { historyIndex: 2, textOffset: 0 }); +}); + +test("delta-only and mixed histories drain every character exactly once", function () { + [ + [{ type: "delta", text: "😀answer".repeat(2500) }], + [user("question".repeat(1000)), { type: "delta", text: "a".repeat(9000) }, + { type: "tool_result", content: "ignored" }, { type: "delta", text: "😀b".repeat(3000) }, mate("legacy")], + Array.from({ length: 1000 }, function (_, i) { return { type: i % 2 ? "delta" : "user_message", text: "entry" + i }; }), + ].forEach(function (history) { + var position = { historyIndex: 0, textOffset: 0 }; + var actual = ""; + var expected = history.filter(function (entry) { return typeof entry.text === "string"; }) + .map(function (entry) { return entry.text; }).join(""); + var drains = 0; + while (cursor.hasPending(history, position)) { + var chunk = cursor.collectChunk(history, position); + assert.ok(chunk.conversationContent.length <= 6000); + chunk.conversationContent.split("\n").forEach(function (line) { + assert.match(line, /^(User|Mate): /); + var text = line.substring(6); + assert.ok(!/[\uD800-\uDBFF]$/.test(text)); + assert.ok(!/^[\uDC00-\uDFFF]/.test(text)); + actual += text; + }); + assert.notDeepEqual(chunk.cursor, position); + position = chunk.cursor; + assert.ok(++drains < 100); + } + assert.equal(actual, expected); + }); +}); + +var createSessionManager = require("../lib/sessions").createSessionManager; +var attachMessageProcessor = require("../lib/sdk-message-processor").attachMessageProcessor; +var normalizeClaudeEvent = require("../lib/yoke/adapters/claude").contractTestKit.normalizeEvent; + +function integration(t) { + var root = fs.mkdtempSync(path.join(os.tmpdir(), "clay-dm-production-")); + t.after(function () { fs.rmSync(root, { recursive: true, force: true }); }); + var mateDir = path.join(root, "mate"); + fs.mkdirSync(path.join(mateDir, "knowledge"), { recursive: true }); + fs.writeFileSync(path.join(mateDir, "mate.yaml"), "relationship: assistant\n"); + var sm = createSessionManager({ cwd: mateDir, sessionsBase: path.join(root, "sessions"), send: function () {} }); + var requests = []; + var sdk = { + createMentionSession: function (options) { + requests.push({ prompt: options.initialMessage, callbacks: options }); + return Promise.resolve({ + isAlive: function () { return true; }, + pushMessage: function (prompt, callbacks) { requests.push({ prompt: prompt, callbacks: callbacks }); }, + }); + }, + }; + var api = attachMateInteraction({ + cwd: mateDir, sm: sm, sdk: sdk, + matesModule: { + buildMateCtx: function () { return {}; }, + isMate: function () { return true; }, + getMateDir: function () { return mateDir; }, + getMate: function () { return { name: "Mate" }; }, + }, + isMate: true, projectOwnerId: "owner", + updateMemorySummary: function () {}, + initMemorySummary: function (_ctx, _id, callback) { callback(); }, + }); + function session(id) { + var value = { + localId: id, _sessionRecordId: "dm-" + id, title: "DM", history: [], + messageUUIDs: [], blocks: {}, sentToolResults: {}, pendingPermissions: {}, + pendingElicitations: {}, pendingAskUser: {}, activeTaskToolIds: {}, taskIdMap: {}, + turnCount: 0, responsePreview: "", + }; + assert.equal(sm.saveSessionFile(value), true); + return value; + } + function disk(value) { + var files = []; + function visit(dir) { + fs.readdirSync(dir, { withFileTypes: true }).forEach(function (entry) { + var target = path.join(dir, entry.name); + if (entry.isDirectory()) visit(target); + else if (entry.name === value._sessionRecordId + ".jsonl") files.push(target); + }); + } + visit(path.join(root, "sessions")); + assert.equal(files.length, 1); + return files[0]; + } + var mp = attachMessageProcessor({ + sm: sm, send: function () {}, onProcessingChanged: function () {}, + onTurnDone: function (value, preview) { api.digestDmTurn(value, preview); }, + }); + return { + root: root, mateDir: mateDir, sm: sm, api: api, mp: mp, requests: requests, session: session, disk: disk, + finish: function (index) { + requests[index].callbacks.onDelta(JSON.stringify({ topic: "test", summary: "saved" })); + requests[index].callbacks.onDone(); + }, + content: function (index) { + return requests[index].prompt.split("Conversation (dm):\n")[1].split("\n\nSTEP 1:")[0]; + }, + }; +} + +function tick() { return new Promise(function (resolve) { setImmediate(resolve); }); } + +function sdkTurn(f, session, text, streamed) { + assert.equal(f.sm.sendAndRecord(session, user("question")), true); + if (streamed) { + [text.slice(0, 17), text.slice(17)].forEach(function (part) { + f.mp.processSDKMessage(session, normalizeClaudeEvent({ type: "stream_event", event: { + type: "content_block_delta", index: 0, delta: { type: "text_delta", text: part }, + } })); + }); + } + f.mp.processSDKMessage(session, normalizeClaudeEvent({ type: "assistant", + message: { content: [{ type: "text", text: text }] }, + })); + f.mp.processSDKMessage(session, normalizeClaudeEvent({ type: "result", subtype: "success", + is_error: false, num_turns: 1, result: text, total_cost_usd: 0.01, + })); +} + +test("SDK streamed and fallback output persist before turn completion and drain fully", async function (t) { + for (var streamed of [true, false]) { + var f = integration(t); + var session = f.session(1); + var answer = "😀full answer ".repeat(1600); + sdkTurn(f, session, answer, streamed); + var persisted = fs.readFileSync(f.disk(session), "utf8").trim().split("\n").map(function (line) { return JSON.parse(line); }); + assert.equal(persisted.filter(function (entry) { return entry.type === "delta"; }).map(function (entry) { return entry.text; }).join(""), answer); + assert.equal(persisted.some(function (entry) { return entry.type === "digest_checkpoint"; }), false); + assert.equal(persisted[persisted.length - 1].type, "done"); + var actual = ""; + var index = 0; + while (index < f.requests.length) { + var content = f.content(index); + assert.ok(content.length <= 6000); + actual += content.split("\n").map(function (line) { assert.match(line, /^(User|Mate): /); return line.slice(6); }).join(""); + f.finish(index++); + await tick(); + } + assert.equal(actual, "question" + answer); + assert.ok(index > 3); + assert.equal(cursor.hasPending(session.history, cursor.readCursor(session.history)), false); + } +}); + +test("real journal append failure leaves input retryable", async function (t) { + var f = integration(t); + var session = f.session(1); + f.sm.sendAndRecord(session, user("retry this")); + var journal = path.join(f.mateDir, "knowledge", "session-digests.jsonl"); + fs.mkdirSync(journal); + f.api.digestDmTurn(session); + f.finish(0); + assert.deepEqual(cursor.readCursor(session.history), { historyIndex: 0, textOffset: 0 }); + assert.equal(session.history.length, 1); + fs.rmdirSync(journal); + await tick(); + f.api.digestDmTurn(session); + assert.equal(f.content(1), f.content(0)); + f.finish(1); + assert.equal(session.history.filter(function (entry) { return entry.type === "digest_checkpoint"; }).length, 1); +}); + +test("real checkpoint append failure retries and can duplicate an already saved journal", async function (t) { + var f = integration(t); + var session = f.session(1); + f.sm.sendAndRecord(session, user("keep this")); + var file = f.disk(session); + var saved = fs.readFileSync(file); + f.api.digestDmTurn(session); + fs.unlinkSync(file); + fs.mkdirSync(file); + f.finish(0); + assert.deepEqual(cursor.readCursor(session.history), { historyIndex: 0, textOffset: 0 }); + assert.equal(session.history.length, 1); + fs.rmdirSync(file); + fs.writeFileSync(file, saved); + await tick(); + f.api.digestDmTurn(session); + assert.equal(f.content(1), f.content(0)); + f.finish(1); + var journal = fs.readFileSync(path.join(f.mateDir, "knowledge", "session-digests.jsonl"), "utf8").trim().split("\n"); + assert.equal(journal.length, 2, "separate journal/checkpoint writes permit duplicate digests on retry"); + assert.equal(cursor.readCursor(session.history).historyIndex, 1); +}); + +test("restart restores persisted within-delta offset and drains the remainder", async function (t) { + var f = integration(t); + var session = f.session(1); + var text = "😀restart ".repeat(1500); + f.sm.sendAndRecord(session, { type: "delta", text: text }); + f.api.digestDmTurn(session); + var first = f.content(0).slice(6); + f.finish(0); + session.history = fs.readFileSync(f.disk(session), "utf8").trim().split("\n").map(function (line) { return JSON.parse(line); }).filter(function (entry) { return entry.type !== "meta"; }); + // Session persistence has a leading metadata record, outside history. + var offset = cursor.readCursor(session.history); + assert.equal(offset.historyIndex, 0); + assert.equal(offset.textOffset, first.length); + var restored = Object.assign({}, session, { history: session.history.slice() }); + // Retire the old in-memory history before its scheduled drain, as on shutdown. + session.history = []; + var f2 = integration(t); + assert.equal(f2.sm.saveSessionFile(restored), true); + f2.api.digestDmTurn(restored); + var actual = first; + for (var i = 0; i < f2.requests.length; i++) { + actual += f2.content(i).slice(6); + f2.finish(i); + await tick(); + } + assert.equal(actual, text); + await tick(); +}); + +test("in-flight arrivals and a second DM session drain without another user message", async function (t) { + var f = integration(t); + var a = f.session(1); + var b = f.session(2); + f.sm.sendAndRecord(a, user("first")); + f.api.digestDmTurn(a); + await tick(); + f.sm.sendAndRecord(a, { type: "delta", text: "arrived asynchronously" }); + f.api.digestDmTurn(a); + f.sm.sendAndRecord(b, { type: "delta", text: "other session" }); + f.api.digestDmTurn(b); + assert.equal(f.requests.length, 1); + f.finish(0); + await tick(); + assert.equal(f.content(1), "Mate: other session"); + f.finish(1); + await tick(); + assert.equal(f.content(2), "Mate: arrived asynchronously"); + f.finish(2); + await tick(); + assert.equal(f.requests.length, 3); + assert.equal(cursor.hasPending(a.history, cursor.readCursor(a.history)), false); + assert.equal(cursor.hasPending(b.history, cursor.readCursor(b.history)), false); +}); diff --git a/test/fixtures/knowledge-db-legacy-aggregate.json b/test/fixtures/knowledge-db-legacy-aggregate.json new file mode 100644 index 00000000..e925fca3 --- /dev/null +++ b/test/fixtures/knowledge-db-legacy-aggregate.json @@ -0,0 +1,426 @@ +{ + "sourceCommit": "76abe088", + "clock": 1791849600000, + "originalImportArgs": { + "databaseId": "db_3c18967f-66e0-4a2b-872a-7568694d033c", + "expectedDatabaseRevision": 1, + "operationId": "legacy_1791849600000_2", + "sourceNamespace": "legacy", + "rows": [ + { + "sourceKey": "one", + "sourcePayload": { + "original": "one" + }, + "values": { + "fld_title00": "One", + "fld_notes00": "Original\n\ttext", + "fld_amount0": 7 + } + }, + { + "sourceKey": "two", + "sourcePayload": { + "original": "two" + }, + "values": { + "fld_title00": "Two", + "fld_amount0": 11 + } + } + ] + }, + "firstId": "rec_ee5cbe41-4773-4889-bb0f-2c4c18767fb3", + "secondId": "rec_200e02d5-707f-48fb-9cdb-c4c2623ba615", + "databases": [ + { + "_id": "db_3c18967f-66e0-4a2b-872a-7568694d033c", + "name": "Legacy renamed", + "revision": 5, + "archivedAt": null, + "schema": [ + { + "id": "fld_title00", + "name": "Title", + "type": "title", + "required": true, + "archived": false + }, + { + "id": "fld_notes00", + "name": "Notes", + "type": "text", + "required": false, + "archived": false + }, + { + "id": "fld_amount0", + "name": "Amount", + "type": "number", + "required": false, + "archived": false + } + ], + "views": [ + { + "id": "view_926533e0-0b82-48b4-a3fa-9e6bb0d10377", + "name": "Table", + "type": "table", + "columns": [ + "fld_title00", + "fld_notes00", + "fld_amount0" + ], + "filters": [], + "sorts": [], + "archived": false + } + ], + "records": [ + { + "id": "rec_ee5cbe41-4773-4889-bb0f-2c4c18767fb3", + "revision": 2, + "values": { + "fld_title00": "One", + "fld_notes00": "Human correction", + "fld_amount0": 7 + }, + "archivedAt": null, + "createdAt": 1791849600000, + "updatedAt": 1791849600000, + "createdBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "updatedBy": { + "type": "user", + "userId": null, + "displayName": null + } + }, + { + "id": "rec_200e02d5-707f-48fb-9cdb-c4c2623ba615", + "revision": 2, + "values": { + "fld_title00": "Two", + "fld_amount0": 11 + }, + "archivedAt": 1791849600000, + "createdAt": 1791849600000, + "updatedAt": 1791849600000, + "createdBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "updatedBy": { + "type": "user", + "userId": null, + "displayName": null + } + } + ], + "recordHistory": { + "rec_ee5cbe41-4773-4889-bb0f-2c4c18767fb3": [ + { + "revision": 1, + "at": 1791849600000, + "actor": { + "type": "user", + "userId": null, + "displayName": null + }, + "action": "update", + "values": { + "fld_title00": "One", + "fld_notes00": "Original\n\ttext", + "fld_amount0": 7 + } + } + ], + "rec_200e02d5-707f-48fb-9cdb-c4c2623ba615": [ + { + "revision": 1, + "at": 1791849600000, + "actor": { + "type": "user", + "userId": null, + "displayName": null + }, + "action": "archive", + "values": { + "fld_title00": "Two", + "fld_amount0": 11 + }, + "archivedAt": null + } + ] + }, + "databaseHistory": [ + { + "revision": 4, + "at": 1791849600000, + "actor": { + "type": "user", + "userId": null, + "displayName": null + }, + "action": "rename", + "name": "Legacy" + } + ], + "receipts": [ + { + "operationId": "legacy_1791849600000_1", + "fingerprint": "77ea192d1941186193592b7c9cdaebd473b47265489074d8e5fad9d770c2d913", + "at": 1791849600000, + "result": { + "database": { + "id": "db_3c18967f-66e0-4a2b-872a-7568694d033c", + "itemType": "db", + "name": "Legacy", + "revision": 1, + "archivedAt": null, + "createdAt": 1791849600000, + "updatedAt": 1791849600000, + "createdBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "updatedBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "schema": [ + { + "id": "fld_title00", + "name": "Title", + "type": "title", + "required": true, + "archived": false + }, + { + "id": "fld_notes00", + "name": "Notes", + "type": "text", + "required": false, + "archived": false + }, + { + "id": "fld_amount0", + "name": "Amount", + "type": "number", + "required": false, + "archived": false + } + ], + "views": [ + { + "id": "view_926533e0-0b82-48b4-a3fa-9e6bb0d10377", + "name": "Table", + "type": "table", + "columns": [ + "fld_title00", + "fld_notes00", + "fld_amount0" + ], + "filters": [], + "sorts": [], + "archived": false + } + ], + "recordCount": 0, + "projectionPending": true, + "projectionError": null + } + } + }, + { + "operationId": "legacy_1791849600000_2", + "fingerprint": "d2d93e6f6e9ba4ca194fe6bac9a97bd5cb2ff59445cce8522fb932e2aa0cf061", + "at": 1791849600000, + "result": { + "databaseId": "db_3c18967f-66e0-4a2b-872a-7568694d033c", + "databaseRevision": 2, + "sourceNamespace": "legacy", + "createdCount": 2, + "replayedCount": 0, + "mappings": [ + { + "sourceNamespace": "legacy", + "sourceKey": "one", + "fingerprint": "9a2a62d552e8c7f892e927df94800c81aa127b6a5bf8bc69df78fba787c2db52", + "recordId": "rec_ee5cbe41-4773-4889-bb0f-2c4c18767fb3", + "created": true + }, + { + "sourceNamespace": "legacy", + "sourceKey": "two", + "fingerprint": "7e515deff58d7b01bfbb321d822360fcc7551f1980d49ffecdc8728708cb4c1c", + "recordId": "rec_200e02d5-707f-48fb-9cdb-c4c2623ba615", + "created": true + } + ] + } + }, + { + "operationId": "legacy_1791849600000_3", + "fingerprint": "dea0ca665b731e2e81aa81586b13fd32d0561e3841497f5163a8c2dc017f88a2", + "at": 1791849600000, + "result": { + "databaseRevision": 3, + "record": { + "id": "rec_ee5cbe41-4773-4889-bb0f-2c4c18767fb3", + "revision": 2, + "values": { + "fld_title00": "One", + "fld_notes00": "Human correction", + "fld_amount0": 7 + }, + "archivedAt": null, + "createdAt": 1791849600000, + "updatedAt": 1791849600000, + "createdBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "updatedBy": { + "type": "user", + "userId": null, + "displayName": null + } + } + } + }, + { + "operationId": "legacy_1791849600000_4", + "fingerprint": "cd0f1740fbc4a1ff79564b230d5b4fb6640c1b147e4a6ec96bd00b47ac50fb09", + "at": 1791849600000, + "result": { + "databaseRevision": 4, + "record": { + "id": "rec_200e02d5-707f-48fb-9cdb-c4c2623ba615", + "revision": 2, + "values": { + "fld_title00": "Two", + "fld_amount0": 11 + }, + "archivedAt": 1791849600000, + "createdAt": 1791849600000, + "updatedAt": 1791849600000, + "createdBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "updatedBy": { + "type": "user", + "userId": null, + "displayName": null + } + } + } + }, + { + "operationId": "legacy_1791849600000_5", + "fingerprint": "3a8ae946f34ee7e2b5ac66b6a2439452e70f6c62daaa53f9b941b4544938a265", + "at": 1791849600000, + "result": { + "database": { + "id": "db_3c18967f-66e0-4a2b-872a-7568694d033c", + "itemType": "db", + "name": "Legacy renamed", + "revision": 5, + "archivedAt": null, + "createdAt": 1791849600000, + "updatedAt": 1791849600000, + "createdBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "updatedBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "schema": [ + { + "id": "fld_title00", + "name": "Title", + "type": "title", + "required": true, + "archived": false + }, + { + "id": "fld_notes00", + "name": "Notes", + "type": "text", + "required": false, + "archived": false + }, + { + "id": "fld_amount0", + "name": "Amount", + "type": "number", + "required": false, + "archived": false + } + ], + "views": [ + { + "id": "view_926533e0-0b82-48b4-a3fa-9e6bb0d10377", + "name": "Table", + "type": "table", + "columns": [ + "fld_title00", + "fld_notes00", + "fld_amount0" + ], + "filters": [], + "sorts": [], + "archived": false + } + ], + "recordCount": 1, + "projectionPending": false, + "projectionError": null + } + } + } + ], + "createdAt": 1791849600000, + "updatedAt": 1791849600000, + "createdBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "updatedBy": { + "type": "user", + "userId": null, + "displayName": null + }, + "projectionPending": false, + "projectionError": null, + "importMappings": [ + { + "sourceNamespace": "legacy", + "sourceKey": "one", + "fingerprint": "9a2a62d552e8c7f892e927df94800c81aa127b6a5bf8bc69df78fba787c2db52", + "recordId": "rec_ee5cbe41-4773-4889-bb0f-2c4c18767fb3" + }, + { + "sourceNamespace": "legacy", + "sourceKey": "two", + "fingerprint": "7e515deff58d7b01bfbb321d822360fcc7551f1980d49ffecdc8728708cb4c1c", + "recordId": "rec_200e02d5-707f-48fb-9cdb-c4c2623ba615" + } + ] + } + ] +} diff --git a/test/fixtures/knowledge-db-memory-adapter.cjs b/test/fixtures/knowledge-db-memory-adapter.cjs new file mode 100644 index 00000000..4aca1bce --- /dev/null +++ b/test/fixtures/knowledge-db-memory-adapter.cjs @@ -0,0 +1,36 @@ +// Independent contract fixture: no filesystem, NeDB, adapter factory or production helper. +function clone(value) { return JSON.parse(JSON.stringify(value)); } +function createMemoryStorage(state) { + state = state || { values: new Map(), scopes: new Map(), ready: false }; + function initialize() { state.ready = true; return Promise.resolve(); } + function serialize(scope, work) { + var previous = state.scopes.get(scope) || Promise.resolve(); + var next = previous.catch(function () {}).then(function () { return work(); }); + state.scopes.set(scope, next.catch(function () {})); return next; + } + return { + contractVersion: 1, + initialize: initialize, hasStore: function () { return state.ready; }, isReady: function () { return state.ready; }, + get: function (id) { return initialize().then(function () { return state.values.has(id) ? clone(state.values.get(id)) : null; }); }, + list: function () { return initialize().then(function () { return Array.from(state.values.values()).map(clone); }); }, + getCached: function () { return Array.from(state.values.values()).map(clone); }, + insert: function (snapshot) { + var value = clone(snapshot); + return initialize().then(function () { + if (state.values.has(value._id)) throw Object.assign(new Error("Identity exists."), { code: "ALREADY_EXISTS" }); + state.values.set(value._id, value); + }); + }, + replaceIfRevision: function (id, revision, snapshot) { + var value = clone(snapshot); + if (value._id !== id) return Promise.reject(Object.assign(new Error("Identity mismatch."), { code: "INVALID_STORAGE_ID" })); + return initialize().then(function () { + var current = state.values.get(id); + if (!current || current.revision !== revision) return false; + state.values.set(id, value); return true; + }); + }, + serialize: serialize + }; +} +module.exports = { createMemoryStorage: createMemoryStorage }; diff --git a/test/knowledge-db-import-storage.test.js b/test/knowledge-db-import-storage.test.js new file mode 100644 index 00000000..48bee48a --- /dev/null +++ b/test/knowledge-db-import-storage.test.js @@ -0,0 +1,130 @@ +var test = require("node:test"); +var assert = require("node:assert/strict"); +var fs = require("fs"); +var os = require("os"); +var path = require("path"); +var child = require("child_process"); +var Datastore = require("@seald-io/nedb"); +var storageModule = require("../lib/knowledge-db-storage-nedb-maintenance"); +var serviceModule = require("../lib/knowledge-db-service"); +var sequence = 0; +function op() { return "storage_" + Date.now() + "_" + (++sequence); } +function deferred() { var resolve; var promise = new Promise(function (done) { resolve = done; }); return { promise: promise, resolve: resolve }; } +function fixture(t, settings) { + var root = fs.mkdtempSync(path.join(os.tmpdir(), "clay-db-storage-")); + t.after(function () { fs.rmSync(root, { recursive: true, force: true }); }); + return { root: root, filename: path.join(root, "knowledge", ".clay-db", "databases.db"), service: serviceModule.createKnowledgeDbService(Object.assign({ projectDir: root }, settings)) }; +} +async function database(service, name) { + return (await service.create({ name: name, schema: [{ id: "fld_title00", name: "Title", type: "title", required: true }, { id: "fld_notes00", name: "Notes", type: "text" }], operationId: op() })).database; +} +function rows(offset, count, narrative) { + var out = []; + for (var i = offset; i < offset + count; i++) { var values = { fld_title00: "Synthetic " + i }; if (narrative) values.fld_notes00 = "n".repeat(narrative); out.push({ sourceKey: String(i), sourcePayload: { original: values }, values: values }); } + return out; +} +async function args(service, db, batch) { return { databaseId: db.id, expectedDatabaseRevision: (await service.read({ databaseId: db.id })).database.revision, sourceNamespace: "synthetic", rows: batch, operationId: op() }; } +async function seed(service, db) { for (var i = 0; i < 300; i += 100) await service.importRecords(await args(service, db, rows(i, 100, 1000))); } +function durableReplay(root, requests) { + var script = 'var assert=require("assert/strict");var service=require(process.argv[1]).createKnowledgeDbService({projectDir:process.argv[2]});var requests=JSON.parse(process.argv[3]);(async function(){assert.equal(await service.initialize(),undefined);assert.equal((await service.readStorageStatus()).automaticCompaction,false);assert.equal(service.listCached({}).databases.length,requests.length);for(var request of requests){var result=await service.importRecords(request);assert.equal(result.createdCount,1);assert.equal((await service.query({databaseId:request.databaseId})).totals.count,1);assert.equal((await service.readImportProgress({databaseId:request.databaseId,sourceNamespace:"synthetic"})).total,1);}process.stdout.write("durable");})().catch(function(e){console.error(e);process.exit(1);});'; + assert.equal(child.execFileSync(process.execPath, ["-e", script, path.resolve(__dirname, "../lib/knowledge-db-service.js"), root, JSON.stringify(requests)], { encoding: "utf8" }), "durable"); +} + +test("shared storage gate holds failed-write reload ahead of concurrent writes and reads to another database", async function (t) { + var f = fixture(t, { projectSummary: function () {} }); var s = f.service; var a = await database(s, "A"); var b = await database(s, "B"); + // A second service facade must share exactly the same storage gate. + var other = serviceModule.createKnowledgeDbService({ projectDir: f.root, projectSummary: function () {} }); + var entered = deferred(); var release = deferred(); var events = []; var originalUpdate = Datastore.prototype.updateAsync; var originalLoad = Datastore.prototype.loadDatabaseAsync; var fail = true; + Datastore.prototype.updateAsync = function (query, value, options) { + if (this.filename !== f.filename) return originalUpdate.call(this, query, value, options); + events.push("write:" + query._id); + if (query._id !== a.id || !fail) return originalUpdate.call(this, query, value, options); + fail = false; var persistence = this.persistence; var originalPersist = persistence.persistNewStateAsync; + persistence.persistNewStateAsync = async function () { entered.resolve(); await release.promise; throw new Error("injected canonical persistence failure"); }; + return originalUpdate.call(this, query, value, options).finally(function () { persistence.persistNewStateAsync = originalPersist; }); + }; + Datastore.prototype.loadDatabaseAsync = function () { + if (this.filename !== f.filename) return originalLoad.call(this); + events.push("reload:start"); return originalLoad.call(this).then(function (value) { events.push("reload:end"); return value; }); + }; + t.after(function () { release.resolve(); Datastore.prototype.updateAsync = originalUpdate; Datastore.prototype.loadDatabaseAsync = originalLoad; }); + var aArgs = await args(s, a, rows(0, 1)); var bArgs = await args(s, b, rows(0, 1)); + var aWrite = s.importRecords(aArgs); var aOutcome = aWrite.then(function (result) { return { result: result }; }, function (cause) { return { cause: cause }; }); + await entered.promise; + assert.equal(s.listCached({}).databases.find(function (db) { return db.id === a.id; }).recordCount, 0); + var bWrite = other.importRecords(bArgs); var observed = s.query({ databaseId: a.id }); + await new Promise(function (done) { setImmediate(done); }); assert.ok(!events.includes("write:" + b.id)); + release.resolve(); var results = await Promise.all([aOutcome, bWrite, observed]); + assert.match(results[0].cause.message, /injected canonical persistence failure/); assert.equal(results[1].createdCount, 1); assert.equal(results[2].totals.count, 0); + assert.ok(events.indexOf("reload:end") < events.indexOf("write:" + b.id)); + var retried = await s.importRecords(aArgs); assert.equal(retried.createdCount, 1); assert.equal(retried.databaseRevision, 2); + assert.equal((await other.importRecords(bArgs)).mappings[0].recordId, results[1].mappings[0].recordId); + assert.equal((await s.readStorageStatus()).recoveries, 1); + Datastore.prototype.updateAsync = originalUpdate; Datastore.prototype.loadDatabaseAsync = originalLoad; + durableReplay(f.root, [aArgs, bArgs]); +}); + +test("compaction failure preserves canonical successes and retries under the shared gate including projection failure", async function (t) { + var projectionFails = false; var f = fixture(t, { projectSummary: function (summary) { if (projectionFails && summary.name === "A") throw new Error("injected summary failure"); } }); var s = f.service; + var a = await database(s, "A"); var b = await database(s, "B"); await seed(s, a); + var originalCompact = Datastore.prototype.compactDatafileAsync; var originalUpdate = Datastore.prototype.updateAsync; var entered = deferred(); var release = deferred(); var hold = false; var failing = true; var compacting = false; var attempts = 0; + Datastore.prototype.compactDatafileAsync = async function () { + if (this.filename !== f.filename) return originalCompact.call(this); + assert.equal(compacting, false); compacting = true; attempts++; + try { if (hold) { hold = false; entered.resolve(); await release.promise; } if (failing) throw new Error("injected compaction failure"); return await originalCompact.call(this); } + finally { compacting = false; } + }; + Datastore.prototype.updateAsync = function (query, value, options) { if (this.filename === f.filename) assert.equal(compacting, false); return originalUpdate.call(this, query, value, options); }; + t.after(function () { release.resolve(); Datastore.prototype.compactDatafileAsync = originalCompact; Datastore.prototype.updateAsync = originalUpdate; }); + for (var i = 0; i < 8 && !attempts; i++) await s.importRecords(await args(s, a, rows(0, 100, 1000))); + assert.ok(attempts > 0); var failedStatus = await s.readStorageStatus(); assert.equal(failedStatus.maintenancePending, true); assert.match(failedStatus.maintenanceError, /injected compaction failure/); + var aArgs = await args(s, a, rows(300, 1, 1000)); var bArgs = await args(s, b, rows(0, 1)); + hold = true; projectionFails = true; var aWrite = s.importRecords(aArgs); await entered.promise; var bWrite = s.importRecords(bArgs); + await new Promise(function (done) { setImmediate(done); }); assert.equal(compacting, true); release.resolve(); + var results = await Promise.all([aWrite, bWrite]); assert.equal(results[0].createdCount, 1); assert.equal(results[0].projectionPending, true); assert.match(results[0].projectionError, /injected summary failure/); assert.equal(results[1].createdCount, 1); + assert.equal((await s.query({ databaseId: a.id })).totals.count, 301); assert.equal((await s.query({ databaseId: b.id })).totals.count, 1); + assert.equal((await s.importRecords(aArgs)).mappings[0].recordId, results[0].mappings[0].recordId); + failing = false; projectionFails = false; + var retried = await s.retryProjection({ databaseId: a.id }); assert.equal(retried.projectionPending, false); + var status = await s.readStorageStatus(); assert.equal(status.maintenancePending, false); assert.equal(status.maintenanceError, null); assert.ok(status.compactions > 0); assert.ok(status.failedCompactions >= 2); + assert.ok(status.journalBytes <= Math.max(storageModule.MIN_COMPACTION_BYTES, status.liveBytes * 4)); +}); + +test("persistent compaction failure bounds further growth by rejecting before the next canonical write", async function (t) { + var f = fixture(t); var s = f.service; var db = await database(s, "Bounded failure"); await seed(s, db); + var originalCompact = Datastore.prototype.compactDatafileAsync; + Datastore.prototype.compactDatafileAsync = function () { if (this.filename === f.filename) return Promise.reject(new Error("maintenance unavailable")); return originalCompact.call(this); }; + t.after(function () { Datastore.prototype.compactDatafileAsync = originalCompact; }); + var blockedArgs; var priorWrites; + for (var i = 0; i < 20; i++) { + var request = await args(s, db, rows(0, 100, 1000)); priorWrites = (await s.readStorageStatus()).successfulWrites; + try { var result = await s.importRecords(request); assert.equal(result.createdCount, 0); } + catch (cause) { assert.equal(cause.code, "STORAGE_MAINTENANCE_BLOCKED"); blockedArgs = request; break; } + } + assert.ok(blockedArgs); var status = await s.readStorageStatus(); assert.equal(status.successfulWrites, priorWrites); assert.equal(status.maintenancePending, true); + assert.ok(status.journalBytes <= Math.max(storageModule.FAILURE_HEADROOM_BYTES, storageModule.FAILURE_AMPLIFICATION * status.liveBytes) + status.liveBytes); + var previousSize = status.journalBytes; + await assert.rejects(s.importRecords(blockedArgs), function (cause) { return cause.code === "STORAGE_MAINTENANCE_BLOCKED"; }); assert.equal((await s.readStorageStatus()).journalBytes, previousSize); + Datastore.prototype.compactDatafileAsync = originalCompact; + var resumed = await s.importRecords(blockedArgs); assert.equal(resumed.createdCount, 0); assert.equal((await s.query({ databaseId: db.id })).totals.count, 300); assert.equal((await s.readStorageStatus()).maintenancePending, false); +}); + +test("failed crash-safe compaction leaves canonical data and receipts durable despite a stale temporary file", async function (t) { + var libraryStorage = require("@seald-io/nedb/lib/storage"); var originalRewrite = libraryStorage.crashSafeWriteFileLinesAsync; + var f = fixture(t, { projectSummary: function () {} }); var s = f.service; var a = await database(s, "A"); var b = await database(s, "B"); await seed(s, a); var failures = 0; + libraryStorage.crashSafeWriteFileLinesAsync = async function (filename, lines, modes) { + if (filename !== f.filename) return originalRewrite.call(this, filename, lines, modes); + failures++; await fs.promises.writeFile(filename + "~", lines.join("\n") + "\n"); throw new Error("injected failure before compaction rename"); + }; + t.after(function () { libraryStorage.crashSafeWriteFileLinesAsync = originalRewrite; }); + for (var i = 0; i < 8 && !failures; i++) { var replay = await s.importRecords(await args(s, a, rows(0, 100, 1000))); assert.equal(replay.createdCount, 0); } + assert.ok(failures); assert.ok(fs.statSync(f.filename + "~").size > 0); assert.equal((await s.readStorageStatus()).maintenancePending, true); + var bArgs = await args(s, b, rows(0, 1)); var added = await s.importRecords(bArgs); assert.equal(added.createdCount, 1); + assert.equal((await s.importRecords(bArgs)).mappings[0].recordId, added.mappings[0].recordId); + var backup = fs.mkdtempSync(path.join(os.tmpdir(), "clay-db-maintenance-backup-")); t.after(function () { fs.rmSync(backup, { recursive: true, force: true }); }); + fs.cpSync(path.join(f.root, "knowledge"), path.join(backup, "knowledge"), { recursive: true }); + var script = 'var assert=require("assert/strict");var service=require(process.argv[1]).createKnowledgeDbService({projectDir:process.argv[2]});var ids=JSON.parse(process.argv[3]);var request=JSON.parse(process.argv[4]);(async function(){assert.equal((await service.query({databaseId:ids[0]})).totals.count,300);assert.equal((await service.query({databaseId:ids[1]})).totals.count,1);assert.equal((await service.readImportProgress({databaseId:ids[0],sourceNamespace:"synthetic"})).total,300);assert.equal((await service.importRecords(request)).createdCount,1);assert.equal((await service.readStorageStatus()).maintenancePending,false);process.stdout.write("restored");})().catch(function(e){console.error(e);process.exit(1);});'; + assert.equal(child.execFileSync(process.execPath, ["-e", script, path.resolve(__dirname, "../lib/knowledge-db-service.js"), backup, JSON.stringify([a.id, b.id]), JSON.stringify(bArgs)], { encoding: "utf8" }), "restored"); + libraryStorage.crashSafeWriteFileLinesAsync = originalRewrite; + await s.retryProjection({ databaseId: a.id }); assert.equal((await s.readStorageStatus()).maintenancePending, false); assert.equal(fs.existsSync(f.filename + "~"), false); +}); diff --git a/test/knowledge-db-import.test.js b/test/knowledge-db-import.test.js new file mode 100644 index 00000000..f3a29c8f --- /dev/null +++ b/test/knowledge-db-import.test.js @@ -0,0 +1,210 @@ +var test = require("node:test"); +var assert = require("node:assert/strict"); +var fs = require("fs"); +var os = require("os"); +var path = require("path"); +var child = require("child_process"); +var Datastore = require("@seald-io/nedb"); +var serviceModule = require("../lib/knowledge-db-service"); +var imports = require("../lib/knowledge-db-import"); +var storageModule = require("../lib/knowledge-db-storage-nedb-maintenance"); +var sequence = 0; +function op() { return "import_" + Date.now() + "_" + (++sequence); } +function fixture(t, options) { + var root = fs.mkdtempSync(path.join(os.tmpdir(), "clay-import-")); + t.after(function () { fs.rmSync(root, { recursive: true, force: true }); }); + return { root: root, filename: path.join(root, "knowledge", ".clay-db", "databases.db"), service: serviceModule.createKnowledgeDbService(Object.assign({ projectDir: root }, options)) }; +} +function schema(extra) { return [{ id: "fld_title00", name: "Title", type: "title", required: true }].concat(extra || []); } +async function database(service, name, extra) { return (await service.create({ name: name, schema: schema(extra), operationId: op() })).database; } +function row(key, values) { return { sourceKey: String(key), sourcePayload: { key: String(key), original: values }, values: values }; } +async function request(service, db, rows, namespace) { + var revision = (await service.read({ databaseId: db.id })).database.revision; + return service.importRecords({ databaseId: db.id, expectedDatabaseRevision: revision, sourceNamespace: namespace || "synthetic", rows: rows, operationId: op() }); +} +async function rejects(work, code) { await assert.rejects(async function () { return work(); }, function (cause) { return cause.code === code; }); } +async function scan(service, db, count) { + var cursor = null; var ids = new Set(); + do { var page = await service.query({ databaseId: db.id, limit: 100, cursor: cursor }); assert.equal(page.totals.count, count); page.records.forEach(function (record) { assert.ok(!ids.has(record.id)); ids.add(record.id); }); cursor = page.nextCursor; } while (cursor); + assert.equal(ids.size, count); +} +function isolated(script, args) { + return JSON.parse(child.execFileSync(process.execPath, ["-e", script, path.resolve(__dirname, "../lib/knowledge-db-service.js")].concat(args), { encoding: "utf8", maxBuffer: 1024 * 1024 })); +} + +test("production import covers vocabulary, finance relationships, exact scans and isolated restart/backup replay", async function (t) { + var f = fixture(t, { projectSummary: function () {} }); var service = f.service; var writes = 0; var aggregateBytes = 0; var original = Datastore.prototype.updateAsync; + Datastore.prototype.updateAsync = function (query, value, options) { if (this.filename === f.filename) { writes++; aggregateBytes = Math.max(aggregateBytes, Buffer.byteLength(JSON.stringify(value))); } return original.call(this, query, value, options); }; + t.after(function () { Datastore.prototype.updateAsync = original; }); + var vocabulary = await database(service, "Vocabulary", [{ id: "fld_notes00", name: "Definition", type: "text" }]); + var initialWrites = writes; var start = Date.now(); var firstRows; var firstResult; var firstRequest; + for (var offset = 0; offset < 8563; offset += 100) { + var rows = []; + for (var i = offset; i < Math.min(offset + 100, 8563); i++) rows.push(row(i, { fld_title00: "Synthetic word " + i, fld_notes00: "A synthetic definition with context.\n\tUsage example " + i + ": " + "meaning ".repeat(20) })); + var revision = (await service.read({ databaseId: vocabulary.id })).database.revision; + var args = { databaseId: vocabulary.id, expectedDatabaseRevision: revision, sourceNamespace: "vocabulary", rows: rows, operationId: op() }; + var result = await service.importRecords(args); + assert.equal(result.createdCount, rows.length); assert.equal(result.databaseRevision, revision + 1); + if (!offset) { firstRows = rows; firstResult = result; firstRequest = args; } + } + assert.equal(writes - initialWrites, 172); + t.diagnostic("8563 vocabulary aggregate bytes: " + aggregateBytes); + var exactReplay = await service.importRecords(firstRequest); assert.deepEqual(exactReplay.mappings, firstResult.mappings); assert.equal(exactReplay.databaseRevision, firstResult.databaseRevision); assert.equal(writes - initialWrites, 172); + assert.equal((await service.query({ databaseId: vocabulary.id })).totals.count, 8563); + var firstId = firstResult.mappings[0].recordId; + await service.updateRecord({ databaseId: vocabulary.id, recordId: firstId, expectedRevision: 1, values: { fld_notes00: "Human correction\n\tkept" }, operationId: op() }); + await service.archiveRecord({ databaseId: vocabulary.id, recordId: firstResult.mappings[1].recordId, expectedRevision: 1, operationId: op() }); + var beforeReplay = writes; var replay = await request(service, vocabulary, firstRows, "vocabulary"); assert.equal(replay.createdCount, 0); assert.equal(writes, beforeReplay + 1); + assert.equal((await service.readRecord({ databaseId: vocabulary.id, recordId: firstId })).record.values.fld_notes00, "Human correction\n\tkept"); + assert.equal((await service.query({ databaseId: vocabulary.id, archived: true })).totals.count, 1); + await scan(service, vocabulary, 8562); + var progress = await service.readImportProgress({ databaseId: vocabulary.id, sourceNamespace: "vocabulary", limit: 100 }); assert.equal(progress.total, 8563); assert.equal(progress.nextOffset, 100); + var mappingCount = 0; var mappingOffset = 0; + do { var mappings = await service.readImportProgress({ databaseId: vocabulary.id, sourceNamespace: "vocabulary", offset: mappingOffset }); mappingCount += mappings.mappings.length; mappingOffset = mappings.nextOffset; } while (mappingOffset !== null); + assert.equal(mappingCount, 8563); + var categories = await database(service, "Categories"); var spenders = await database(service, "Spenders"); + var category = await request(service, categories, [row("food", { fld_title00: "Food" }), row("travel", { fld_title00: "Travel" })], "categories"); + await request(service, spenders, [row("person", { fld_title00: "Synthetic person" })], "spenders"); + var finance = await database(service, "Transactions", [{ id: "fld_amount0", name: "Amount", type: "number" }, { id: "fld_category", name: "Category", type: "reference", target: "record", targetDatabaseId: categories.id }, { id: "fld_spender", name: "Spender", type: "reference", target: "record", targetDatabaseId: spenders.id }]); + var financeRows = []; var expectedSum = 0; + for (var j = 0; j < 537; j++) { expectedSum += j; financeRows.push(row(j, { fld_title00: "Transaction " + j, fld_amount0: j, fld_category: { type: "record", sourceNamespace: "categories", sourceKey: "food" }, fld_spender: { type: "record", sourceNamespace: "spenders", sourceKey: "person" } })); } + var financeWrites = writes; + for (var n = 0; n < financeRows.length; n += 100) await request(service, finance, financeRows.slice(n, n + 100), "finance"); + assert.equal(writes - financeWrites, 12); await scan(service, finance, 537); + var totals = await service.query({ databaseId: finance.id }); assert.equal(totals.totals.sums.fld_amount0, expectedSum); assert.equal(totals.records[0].references.length, 2); assert.ok(totals.records[0].references.every(function (reference) { return reference.status === "available"; })); + await service.archiveRecord({ databaseId: categories.id, recordId: category.mappings[0].recordId, expectedRevision: 1, operationId: op() }); + assert.equal((await service.query({ databaseId: finance.id })).records[0].references[0].status, "archived"); + var current = (await service.read({ databaseId: vocabulary.id })).database; + var ordinary = await service.createRecord({ databaseId: current.id, expectedDatabaseRevision: current.revision, values: { fld_title00: "Normal CRUD", fld_notes00: "After import" }, operationId: op() }); + var updated = await service.updateRecord({ databaseId: current.id, recordId: ordinary.record.id, expectedRevision: 1, values: { fld_notes00: "Updated" }, operationId: op() }); assert.equal(updated.record.revision, 2); + var storageStatus = await service.readStorageStatus(); + assert.equal(storageStatus.maintenancePending, false); assert.ok(storageStatus.compactions > 0); + assert.ok(storageStatus.journalBytes <= Math.max(storageModule.MIN_COMPACTION_BYTES, storageStatus.liveBytes * 4)); + assert.ok(storageStatus.peakJournalBytes <= Math.max(storageModule.MIN_COMPACTION_BYTES, storageStatus.liveBytes * 4)); + var replayFile = path.join(f.root, "replay.json"); fs.writeFileSync(replayFile, JSON.stringify({ databaseId: vocabulary.id, firstId: firstId, rows: firstRows, oldOperationId: firstRequest.operationId, count: 8563 })); + var restartScript = 'var fs = require("fs"); var assert = require("assert/strict"); var mod = require(process.argv[1]); var data = JSON.parse(fs.readFileSync(process.argv[3])); Date.now = function () { return Number(process.argv[4]); }; (async function () { var s = mod.createKnowledgeDbService({projectDir:process.argv[2]}); var db = (await s.read({databaseId:data.databaseId})).database; var result = await s.importRecords({databaseId:db.id, expectedDatabaseRevision:db.revision, operationId:"resume_"+Date.now()+"_fresh", sourceNamespace:"vocabulary", rows:data.rows}); assert.equal(result.createdCount,0); assert.equal(result.replayedCount,100); assert.equal((await s.read({databaseId:db.id})).database.revision,db.revision); var record = await s.readRecord({databaseId:db.id,recordId:data.firstId}); assert.equal(record.record.values.fld_notes00,"Human correction\\n\\tkept"); assert.equal(record.history.length,1); assert.equal((await s.query({databaseId:db.id})).totals.count,data.count); await assert.rejects(async function(){return s.importRecords({databaseId:db.id, expectedDatabaseRevision:db.revision, operationId:data.oldOperationId, sourceNamespace:"vocabulary", rows:data.rows});},function(e){return e.code==="OPERATION_EXPIRED";}); process.stdout.write(JSON.stringify({count:data.count, replayed:result.replayedCount})); })().catch(function(e){console.error(e);process.exit(1);});'; + var future = String(Date.now() + serviceModule.IDEMPOTENCY_WINDOW_MS + 10000); + assert.equal(isolated(restartScript, [f.root, replayFile, future]).replayed, 100); + var backup = fs.mkdtempSync(path.join(os.tmpdir(), "clay-import-backup-")); t.after(function () { fs.rmSync(backup, { recursive: true, force: true }); }); + fs.cpSync(path.join(f.root, "knowledge"), path.join(backup, "knowledge"), { recursive: true }); + assert.equal(isolated(restartScript, [backup, replayFile, future]).count, 8563); + t.diagnostic(JSON.stringify({ rows: 8563, financeRows: 537, elapsedMs: Date.now() - start, maximumAggregateBytes: aggregateBytes, peakJournalBytes: storageStatus.peakJournalBytes, finalJournalBytes: storageStatus.journalBytes, afterRestartFileBytes: fs.statSync(f.filename).size, liveBytes: storageStatus.liveBytes, compactions: storageStatus.compactions, vocabularyCanonicalWrites: 86, vocabularyProjectionWrites: 86, financeCanonicalWrites: 6, financeProjectionWrites: 6 })); +}); + +test("import validates every row atomically, same-batch forward references, conflicts, revisions and archives", async function (t) { + var f = fixture(t); var s = f.service; var db = await database(s, "References", [{ id: "fld_related", name: "Related", type: "reference", target: "record" }]); + var rows = [row("first", { fld_title00: "First", fld_related: { type: "record", sourceNamespace: "synthetic", sourceKey: "second" } }), row("second", { fld_title00: "Second" })]; + var original = await request(s, db, rows); var detail = await s.readRecord({ databaseId: db.id, recordId: original.mappings[0].recordId }); assert.equal(detail.record.values.fld_related.id, original.mappings[1].recordId); assert.equal(detail.record.references[0].status, "available"); + var revision = (await s.read({ databaseId: db.id })).database.revision; + await rejects(function () { return s.importRecords({ databaseId: db.id, expectedDatabaseRevision: 1, sourceNamespace: "synthetic", rows: rows, operationId: op() }); }, "STALE"); + var changed = JSON.parse(JSON.stringify(rows)); changed[0].sourcePayload.newContent = true; + await rejects(function () { return request(s, db, changed); }, "IMPORT_SOURCE_CONFLICT"); + var changedValues = JSON.parse(JSON.stringify(rows)); changedValues[0].values.fld_title00 = "Changed"; + await rejects(function () { return request(s, db, changedValues); }, "IMPORT_SOURCE_CONFLICT"); + await rejects(function () { return request(s, db, [row("third", { fld_title00: "Third" }), row("invalid", { fld_title00: 42 })]); }, "INVALID"); + await rejects(function () { return request(s, db, [row("third", { fld_title00: "Third" }), row("missing", { fld_title00: "Missing", fld_related: { type: "record", sourceNamespace: "synthetic", sourceKey: "absent" } })]); }, "FORBIDDEN_REFERENCE"); + await rejects(function () { return request(s, db, [row("external", { fld_title00: "External", fld_related: { type: "record", databaseId: "db_outside123", id: "rec_outside1" } })]); }, "FORBIDDEN_REFERENCE"); + assert.equal((await s.read({ databaseId: db.id })).database.revision, revision); assert.equal((await s.readImportProgress({ databaseId: db.id, sourceNamespace: "synthetic" })).total, 2); assert.equal((await s.query({ databaseId: db.id })).totals.count, 2); + var archived = await s.archiveRecord({ databaseId: db.id, recordId: original.mappings[1].recordId, expectedRevision: 1, operationId: op() }); + await request(s, db, rows); assert.ok((await s.readRecord({ databaseId: db.id, recordId: archived.record.id })).record.archivedAt); + var recovered = await s.recoverRecord({ databaseId: db.id, recordId: archived.record.id, expectedRevision: 2, operationId: op() }); assert.equal(recovered.record.archivedAt, null); assert.equal((await s.readRecord({ databaseId: db.id, recordId: archived.record.id })).history.length, 2); + revision = (await s.read({ databaseId: db.id })).database.revision; await s.archive({ databaseId: db.id, expectedRevision: revision, operationId: op() }); + await rejects(function () { return s.importRecords({ databaseId: db.id, expectedDatabaseRevision: revision + 1, sourceNamespace: "synthetic", rows: rows, operationId: op() }); }, "ARCHIVED"); + assert.equal((await s.readImportProgress({ databaseId: db.id, sourceNamespace: "synthetic" })).total, 2); +}); + +test("finite batch, byte and aggregate envelopes and multiline text validation", async function (t) { + var f = fixture(t, { maxAggregateBytes: 8000 }); var s = f.service; var db = await database(s, "Limits", [{ id: "fld_notes00", name: "Text", type: "text" }]); + await rejects(function () { return request(s, db, new Array(101).fill(row("one", { fld_title00: "One" }))); }, "IMPORT_BATCH_LIMIT"); + await rejects(function () { return request(s, db, []); }, "IMPORT_BATCH_LIMIT"); + var huge = row("huge", { fld_title00: "Huge" }); huge.sourcePayload = "x".repeat(imports.MAX_BATCH_BYTES); + await rejects(function () { return request(s, db, [huge]); }, "IMPORT_BYTES_LIMIT"); + var large = row("large", { fld_title00: "Large", fld_notes00: "x".repeat(7000) }); + await assert.rejects(async function () { return request(s, db, [large]); }, function (cause) { return cause.code === "LIMIT" && /8000 byte aggregate/.test(cause.message); }); + await rejects(function () { return request(s, db, [row("nul", { fld_title00: "Nul", fld_notes00: "a\u0000b" })]); }, "INVALID"); + await rejects(function () { return request(s, db, [row("control", { fld_title00: "Control", fld_notes00: "a\u000bb" })]); }, "INVALID"); + await rejects(function () { return request(s, db, [row("title", { fld_title00: "Title\n" })]); }, "INVALID"); + var text = " \n\tNarrative\r\nEnd\t "; var result = await request(s, db, [row("text", { fld_title00: "Text", fld_notes00: text })]); assert.equal((await s.readRecord({ databaseId: db.id, recordId: result.mappings[0].recordId })).record.values.fld_notes00, text); + await rejects(function () { return request(s, db, [row("blank", { fld_title00: " " })]); }, "INVALID"); + var bad = row("json", { fld_title00: "Bad" }); bad.sourcePayload = { number: Infinity }; await rejects(function () { return request(s, db, [bad]); }, "INVALID"); + bad.sourcePayload = {}; bad.sourcePayload.self = bad.sourcePayload; await rejects(function () { return request(s, db, [bad]); }, "INVALID"); + await rejects(function () { return s.readImportProgress({ databaseId: db.id, sourceNamespace: "synthetic", limit: 101 }); }, "INVALID"); + assert.equal((await s.query({ databaseId: db.id })).totals.count, 1); +}); + +test("failed NeDB persistence leaves no in-memory import mappings or receipts and can retry", async function (t) { + var f = fixture(t); var s = f.service; var db = await database(s, "Failure"); var original = Datastore.prototype.updateAsync; var fail = true; + Datastore.prototype.updateAsync = function (query, value, options) { + if (this.filename !== f.filename || !fail) return original.call(this, query, value, options); + fail = false; var persistence = this.persistence; var persist = persistence.persistNewStateAsync; + persistence.persistNewStateAsync = async function () { throw new Error("injected disk failure"); }; + return original.call(this, query, value, options).finally(function () { persistence.persistNewStateAsync = persist; }); + }; + t.after(function () { Datastore.prototype.updateAsync = original; }); + var args = { databaseId: db.id, expectedDatabaseRevision: db.revision, sourceNamespace: "synthetic", rows: [row("one", { fld_title00: "One" })], operationId: op() }; + await assert.rejects(s.importRecords(args), /injected disk failure/); + assert.equal((await s.read({ databaseId: db.id })).database.revision, db.revision); assert.equal((await s.readImportProgress({ databaseId: db.id, sourceNamespace: "synthetic" })).total, 0); assert.equal((await s.query({ databaseId: db.id })).totals.count, 0); + var result = await s.importRecords(args); assert.equal(result.createdCount, 1); assert.equal((await s.importRecords(args)).mappings[0].recordId, result.mappings[0].recordId); +}); + +test("measured 20000 row capacity retains finite bounds and live receipt budget", async function (t) { + var f = fixture(t, { projectSummary: function () {} }); var s = f.service; var db = await database(s, "Capacity"); var start = Date.now(); var writes = 0; var largest = 0; var original = Datastore.prototype.updateAsync; + Datastore.prototype.updateAsync = function (query, value, options) { if (this.filename === f.filename) { writes++; largest = Math.max(largest, Buffer.byteLength(JSON.stringify(value))); } return original.call(this, query, value, options); }; + t.after(function () { Datastore.prototype.updateAsync = original; }); + for (var offset = 0; offset < 20000; offset += 100) { + var rows = []; for (var i = offset; i < offset + 100; i++) rows.push(row(i, { fld_title00: "Synthetic entry " + i })); + try { await request(s, db, rows); } catch (cause) { t.diagnostic(JSON.stringify({ acceptedRows: offset, largestAggregateBytes: largest, elapsedMs: Date.now() - start, nextError: cause.message })); throw cause; } + } + assert.equal(writes, 400); var page = await s.query({ databaseId: db.id }); assert.equal(page.totals.count, 20000); + await rejects(function () { return request(s, db, [row("overflow", { fld_title00: "Overflow" })]); }, "LIMIT"); + var current = (await s.read({ databaseId: db.id })).database; + await rejects(function () { return s.createRecord({ databaseId: db.id, expectedDatabaseRevision: current.revision, values: { fld_title00: "Overflow CRUD" }, operationId: op() }); }, "LIMIT"); + await s.updateRecord({ databaseId: db.id, recordId: page.records[0].id, expectedRevision: 1, values: { fld_title00: "Corrected capacity entry" }, operationId: op() }); + assert.equal((await s.readImportProgress({ databaseId: db.id, sourceNamespace: "synthetic" })).total, 20000); + var storageStatus = await s.readStorageStatus(); + assert.equal(storageStatus.maintenancePending, false); assert.ok(storageStatus.compactions > 0); + assert.ok(storageStatus.journalBytes <= Math.max(storageModule.MIN_COMPACTION_BYTES, storageStatus.liveBytes * 4)); + assert.ok(storageStatus.peakJournalBytes <= Math.max(storageModule.MIN_COMPACTION_BYTES, storageStatus.liveBytes * 4)); + t.diagnostic(JSON.stringify({ rows: 20000, canonicalImportWrites: 200, projectionImportWrites: 200, aggregateBytes: largest, elapsedMs: Date.now() - start, peakJournalBytes: storageStatus.peakJournalBytes, finalJournalBytes: storageStatus.journalBytes, liveBytes: storageStatus.liveBytes, compactions: storageStatus.compactions, aggregateLimitBytes: serviceModule.MAX_AGGREGATE_BYTES })); +}); + +test("import survives ambiguous persisted success and keeps bounded summary projection behavior", async function (t) { + var summaries = 0; var f = fixture(t, { projectSummary: function () { summaries++; } }); var s = f.service; var db = await database(s, "Ambiguous failure"); var original = Datastore.prototype.updateAsync; var fail = true; + Datastore.prototype.updateAsync = function (query, value, options) { + if (this.filename !== f.filename || !fail) return original.call(this, query, value, options); + fail = false; var persistence = this.persistence; var persist = persistence.persistNewStateAsync; + persistence.persistNewStateAsync = async function (docs) { await persist.call(this, docs); throw new Error("injected failure after append"); }; + return original.call(this, query, value, options).finally(function () { persistence.persistNewStateAsync = persist; }); + }; + t.after(function () { Datastore.prototype.updateAsync = original; }); + var args = { databaseId: db.id, expectedDatabaseRevision: db.revision, sourceNamespace: "synthetic", rows: [row("one", { fld_title00: "One" })], operationId: op() }; + await assert.rejects(s.importRecords(args), /injected failure after append/); + var retried = await s.importRecords(args); assert.equal(retried.createdCount, 1); assert.equal(retried.databaseRevision, db.revision + 1); assert.equal(retried.projectionPending, true); + assert.equal((await s.query({ databaseId: db.id })).totals.count, 1); assert.equal((await s.readImportProgress({ databaseId: db.id, sourceNamespace: "synthetic" })).total, 1); + assert.equal(summaries, 1); await s.retryProjection({ databaseId: db.id }); assert.equal(summaries, 2); + var writes = 0; Datastore.prototype.updateAsync = function (query, value, options) { if (this.filename === f.filename) writes++; return original.call(this, query, value, options); }; + var batch = []; for (var i = 0; i < 100; i++) batch.push(row("batch-" + i, { fld_title00: "Batch " + i })); + var added = await request(s, db, batch); assert.equal(added.createdCount, 100); assert.equal(writes, 2); assert.equal(summaries, 3); assert.equal(added.projectionPending, false); + await request(s, db, batch); assert.equal(writes, 3); assert.equal(summaries, 3); +}); + +test("imports serialize revision checks, snapshot input, separate namespaces and reject operation reuse", async function (t) { + var f = fixture(t); var s = f.service; var db = await database(s, "Concurrency"); + var args = { databaseId: db.id, expectedDatabaseRevision: 1, sourceNamespace: "first-source", rows: [row("key", { fld_title00: "Original" })], operationId: op() }; + var pending = s.importRecords(args); args.rows[0].values.fld_title00 = "Caller changed input"; + var competing = s.importRecords({ databaseId: db.id, expectedDatabaseRevision: 1, sourceNamespace: "second-source", rows: [row("key", { fld_title00: "Competing" })], operationId: op() }); + var results = await Promise.allSettled([pending, competing]); assert.equal(results[0].status, "fulfilled"); assert.equal(results[1].reason.code, "STALE"); + var first = results[0].value; assert.equal((await s.readRecord({ databaseId: db.id, recordId: first.mappings[0].recordId })).record.values.fld_title00, "Original"); + await rejects(function () { return s.importRecords(args); }, "IDEMPOTENCY_MISMATCH"); + var replayArgs = { databaseId: db.id, expectedDatabaseRevision: 2, sourceNamespace: "first-source", rows: [row("key", { fld_title00: "Original" })], operationId: op() }; + var replay = await s.importRecords(replayArgs); assert.equal(replay.createdCount, 0); assert.equal(replay.databaseRevision, 2); + assert.deepEqual((await s.importRecords(replayArgs)).mappings, replay.mappings); + replayArgs.rows.push(row("extra", { fld_title00: "Extra" })); + await rejects(function () { return s.importRecords(replayArgs); }, "IDEMPOTENCY_MISMATCH"); + var second = await request(s, db, [row("key", { fld_title00: "Other source" })], "second-source"); assert.notEqual(second.mappings[0].recordId, first.mappings[0].recordId); + var mixed = await request(s, db, [row("key", { fld_title00: "Original" }), row("new", { fld_title00: "New" })], "first-source"); assert.equal(mixed.createdCount, 1); assert.equal(mixed.replayedCount, 1); assert.equal(mixed.mappings[0].recordId, first.mappings[0].recordId); + await rejects(function () { return request(s, db, [row("uncommitted", { fld_title00: "Uncommitted" }), row("key", { fld_title00: "Changed source" })], "first-source"); }, "IMPORT_SOURCE_CONFLICT"); + assert.equal((await s.query({ databaseId: db.id })).totals.count, 3); assert.equal((await s.readImportProgress({ databaseId: db.id, sourceNamespace: "first-source" })).total, 2); + await rejects(function () { return request(s, db, [row("duplicate", { fld_title00: "A" }), row("duplicate", { fld_title00: "A" })]); }, "INVALID"); + await rejects(function () { return s.create({ name: "Name\n", operationId: op() }); }, "INVALID"); +}); diff --git a/test/knowledge-db-service.test.js b/test/knowledge-db-service.test.js index 0cf46214..fba1178b 100644 --- a/test/knowledge-db-service.test.js +++ b/test/knowledge-db-service.test.js @@ -29,7 +29,7 @@ test("Knowledge database persists aggregate state, stable ids, history, archive, var recovered = await f.service.recoverRecord({ databaseId: db.id, recordId: one.record.id, expectedRevision: archived.record.revision, operationId: op("recover") }); assert.equal(recovered.record.archivedAt, null); var reopened = require("../lib/knowledge-db-service").createKnowledgeDbService({ projectDir: f.projectDir, authorizeTextReference: function () { return false; } }); var detail = await reopened.readRecord({ databaseId: db.id, recordId: one.record.id }); assert.equal(detail.record.values.fld_number1, 7); assert.equal(detail.history.length, 3); - assert.equal(reopened.filename, path.join(f.projectDir, "knowledge", ".clay-db", "databases.db")); + assert.equal(await reopened.initialize(), undefined); assert.equal(reopened.hasStore(), true); assert.equal(reopened.filename, undefined); }); test("Knowledge database rejects stale concurrent writes, invalid types, unsafe keys, and destructive schema changes", async function (t) { diff --git a/test/knowledge-db-storage-compatibility.test.js b/test/knowledge-db-storage-compatibility.test.js new file mode 100644 index 00000000..02f64060 --- /dev/null +++ b/test/knowledge-db-storage-compatibility.test.js @@ -0,0 +1,37 @@ +var test = require("node:test"); +var assert = require("node:assert/strict"); +var fs = require("fs"); +var os = require("os"); +var path = require("path"); +var child = require("child_process"); +var legacy = require("./fixtures/knowledge-db-legacy-aggregate.json"); +var modulePath = path.resolve(__dirname, "../lib/knowledge-db-service.js"); + +test("default adapter reopens original 76abe088 aggregates without migrating ids, history, receipts or mappings", async function (t) { + var root = fs.mkdtempSync(path.join(os.tmpdir(), "clay-adapter-legacy-read-")); + var backup = fs.mkdtempSync(path.join(os.tmpdir(), "clay-adapter-legacy-backup-")); + t.after(function () { fs.rmSync(root, { recursive: true, force: true }); fs.rmSync(backup, { recursive: true, force: true }); }); + var actualNow = Date.now; Date.now = function () { return legacy.clock; }; t.after(function () { Date.now = actualNow; }); + var directory = path.join(root, "knowledge", ".clay-db"); fs.mkdirSync(directory, { recursive: true }); + // These snapshots were generated by the unmodified service at sourceCommit. + // Persist its existing JSON-lines aggregate representation, not a new adapter format. + var filename = path.join(directory, "databases.db"); fs.writeFileSync(filename, legacy.databases.map(function (value) { return JSON.stringify(value); }).join("\n") + "\n"); + assert.equal(legacy.sourceCommit, "76abe088"); + var create = require("../lib/knowledge-db-service").createKnowledgeDbService; + var service = create({ projectDir: root }); assert.equal(service.hasStore(), true); assert.equal(service.isReady(), false); assert.equal(await service.initialize(), undefined); + var adapter = require("../lib/knowledge-db-storage-nedb").createKnowledgeDbNedbStorage({ projectDir: root }); + var before = legacy.databases[0]; assert.deepEqual(await adapter.get(before._id), before); + assert.equal(service.listCached({}).databases[0].recordCount, 1); assert.equal((await service.read({ databaseId: before._id })).database.name, "Legacy renamed"); + var first = await service.readRecord({ databaseId: before._id, recordId: legacy.firstId }); assert.equal(first.record.values.fld_notes00, "Human correction"); assert.equal(first.history.length, 1); assert.equal(first.record.revision, 2); + var second = await service.readRecord({ databaseId: before._id, recordId: legacy.secondId }); assert.ok(second.record.archivedAt); assert.equal(second.history.length, 1); + var replay = await service.importRecords(legacy.originalImportArgs); assert.equal(replay.createdCount, 2); assert.equal(replay.mappings[0].recordId, legacy.firstId); + assert.deepEqual(await adapter.get(before._id), before); + var progress = await service.readImportProgress({ databaseId: before._id, sourceNamespace: "legacy" }); assert.equal(progress.total, 2); assert.deepEqual(progress.mappings.map(function (item) { return item.recordId; }), [legacy.firstId, legacy.secondId]); + var fresh = await service.importRecords(Object.assign({}, legacy.originalImportArgs, { expectedDatabaseRevision: before.revision, operationId: "fresh_" + legacy.clock + "_compatibility" })); assert.equal(fresh.createdCount, 0); assert.equal(fresh.databaseRevision, before.revision); + var corrected = await service.updateRecord({ databaseId: before._id, recordId: legacy.firstId, expectedRevision: 2, values: { fld_notes00: "Second correction" }, operationId: "edit_" + legacy.clock + "_compatibility" }); assert.equal(corrected.record.revision, 3); + var after = await adapter.get(before._id); assert.equal(after.recordHistory[legacy.firstId].length, 2); assert.deepEqual(after.databaseHistory, before.databaseHistory); assert.deepEqual(after.importMappings, before.importMappings); + assert.ok(after.receipts.some(function (receipt) { return receipt.operationId === legacy.originalImportArgs.operationId; })); + fs.cpSync(path.join(root, "knowledge"), path.join(backup, "knowledge"), { recursive: true }); + var script = 'var assert=require("assert/strict");var fixture=JSON.parse(process.argv[3]);Date.now=function(){return fixture.clock;};var service=require(process.argv[1]).createKnowledgeDbService({projectDir:process.argv[2]});(async function(){assert.equal(await service.initialize(),undefined);var record=await service.readRecord({databaseId:fixture.originalImportArgs.databaseId,recordId:fixture.firstId});assert.equal(record.record.values.fld_notes00,"Second correction");assert.equal(record.history.length,2);assert.ok((await service.readRecord({databaseId:fixture.originalImportArgs.databaseId,recordId:fixture.secondId})).record.archivedAt);var replay=await service.importRecords(fixture.originalImportArgs);assert.equal(replay.createdCount,2);assert.equal(replay.mappings[0].recordId,fixture.firstId);process.stdout.write("compatible");})().catch(function(e){console.error(e);process.exit(1);});'; + assert.equal(child.execFileSync(process.execPath, ["-e", script, modulePath, backup, JSON.stringify(legacy)], { encoding: "utf8" }), "compatible"); +}); diff --git a/test/knowledge-db-storage-contract.test.js b/test/knowledge-db-storage-contract.test.js new file mode 100644 index 00000000..a7f6a154 --- /dev/null +++ b/test/knowledge-db-storage-contract.test.js @@ -0,0 +1,134 @@ +var test = require("node:test"); +var assert = require("node:assert/strict"); +var fs = require("fs"); +var os = require("os"); +var path = require("path"); +var child = require("child_process"); +var createService = require("../lib/knowledge-db-service").createKnowledgeDbService; +var factory = require("../lib/knowledge-db-storage"); +var createMemoryStorage = require("./fixtures/knowledge-db-memory-adapter.cjs").createMemoryStorage; +var sequence = 0; +function op() { return "contract_" + Date.now() + "_" + (++sequence); } +function fixture(t, engine) { + if (engine === "memory") { + var state = { values: new Map(), scopes: new Map(), ready: false }; + return { adapter: createMemoryStorage(state), other: createMemoryStorage(state) }; + } + var root = fs.mkdtempSync(path.join(os.tmpdir(), "clay-adapter-contract-")); + t.after(function () { fs.rmSync(root, { recursive: true, force: true }); }); + var create = require("../lib/knowledge-db-storage-nedb").createKnowledgeDbNedbStorage; + return { adapter: create({ projectDir: root }), other: create({ projectDir: root }) }; +} +function schema(extra) { return [{ id: "fld_title00", name: "Title", type: "title", required: true }].concat(extra || []); } +async function db(service, name, extra) { return (await service.create({ name: name, schema: schema(extra), operationId: op() })).database; } +async function rejection(work, code) { await assert.rejects(async function () { return work(); }, function (cause) { return cause.code === code; }); } + +["memory", "nedb"].forEach(function (engine) { + test(engine + " adapter contract: detached snapshots, unique insertion, atomic revision CAS and shared callback scopes", async function (t) { + var f = fixture(t, engine); var a = f.adapter; var b = f.other; + assert.equal(a.isReady(), false); assert.equal(a.hasStore(), false); assert.deepEqual(a.getCached(), []); + assert.equal(await a.initialize(), undefined); assert.equal(a.isReady(), true); assert.equal(a.hasStore(), true); assert.equal(await a.get("absent"), null); + var value = { _id: "db_contract0", revision: 1, nested: { text: "Original" } }; + var insertion = a.insert(value); value.nested.text = "Caller mutation"; assert.equal(await insertion, undefined); + assert.equal((await b.get(value._id)).nested.text, "Original"); + var read = await a.get(value._id); read.nested.text = "Read mutation"; + var listed = await b.list(); listed[0].nested.text = "List mutation"; + var cached = a.getCached(); cached[0].nested.text = "Cache mutation"; + assert.equal((await a.get(value._id)).nested.text, "Original"); + await rejection(function () { return b.insert(value); }, "ALREADY_EXISTS"); + assert.equal(await a.replaceIfRevision("absent", 1, { _id: "absent", revision: 2 }), false); + await rejection(function () { return a.replaceIfRevision(value._id, 1, { _id: "another", revision: 2 }); }, "INVALID_STORAGE_ID"); + var outcomes = await Promise.all([a.replaceIfRevision(value._id, 1, { _id: value._id, revision: 2, nested: { text: "Winner A" } }), b.replaceIfRevision(value._id, 1, { _id: value._id, revision: 2, nested: { text: "Winner B" } })]); + assert.equal(outcomes.filter(function (result) { return result; }).length, 1); assert.equal((await a.get(value._id)).revision, 2); + var current = await a.get(value._id); current.status = "same revision status"; + assert.equal(await a.replaceIfRevision(value._id, 2, current), true); assert.equal((await b.get(value._id)).status, "same revision status"); + var enter; var release; var entered = new Promise(function (done) { enter = done; }); var barrier = new Promise(function (done) { release = done; }); var events = []; + var first = a.serialize("shared", async function () { assert.equal(arguments.length, 0); enter(); await barrier; events.push("first"); var item = await a.get(value._id); item.status = "serialized"; await a.replaceIfRevision(item._id, item.revision, item); return "first result"; }); + await entered; var second = b.serialize("shared", async function () { events.push("second"); assert.equal((await b.get(value._id)).status, "serialized"); return "second result"; }); + assert.equal(await b.serialize("different", function () { return "different scope"; }), "different scope"); + assert.deepEqual(events, []); release(); assert.deepEqual(await Promise.all([first, second]), ["first result", "second result"]); assert.deepEqual(events, ["first", "second"]); + }); + + test(engine + " injected adapter runs domain CRUD, queries, schema, references, import replay, archive and projection", async function (t) { + var actualNow = Date.now; var clock = actualNow(); Date.now = function () { return clock; }; t.after(function () { Date.now = actualNow; }); + var f = fixture(t, engine); var failProjection = false; var summaries = []; + var service = createService({ storageAdapter: f.adapter, authorizeTextReference: function (id) { return id === "text_allowed1"; }, projectSummary: function (summary) { summaries.push(summary); if (failProjection) throw new Error("synthetic summary outage"); } }); + var other = createService({ storageAdapter: f.other, authorizeTextReference: function (id) { return id === "text_allowed1"; } }); + assert.equal(await service.initialize(), undefined); assert.equal(service.filename, undefined); + var people = await db(service, "People"); var person = await service.createRecord({ databaseId: people.id, expectedDatabaseRevision: 1, values: { fld_title00: "Ada" }, operationId: op() }); + var cases = await db(service, "Cases", [{ id: "fld_notes00", name: "Notes", type: "text" }, { id: "fld_amount0", name: "Amount", type: "number" }, { id: "fld_parent0", name: "Parent", type: "reference", target: "record" }, { id: "fld_person0", name: "Person", type: "reference", target: "record", targetDatabaseId: people.id }, { id: "fld_textref", name: "Brief", type: "reference", target: "text" }]); + var rows = [{ sourceKey: "one", sourcePayload: { original: "one" }, values: { fld_title00: "One", fld_notes00: "Original\n\ttext", fld_amount0: 5, fld_parent0: { type: "record", sourceNamespace: "source", sourceKey: "two" }, fld_person0: { type: "record", id: person.record.id, databaseId: people.id }, fld_textref: { type: "text", id: "text_allowed1" } } }, { sourceKey: "two", sourcePayload: { original: "two" }, values: { fld_title00: "Two", fld_amount0: 3 } }]; + var importArgs = { databaseId: cases.id, expectedDatabaseRevision: 1, operationId: op(), sourceNamespace: "source", rows: rows }; + var imported = await service.importRecords(importArgs); assert.equal(imported.createdCount, 2); assert.equal(imported.databaseRevision, 2); + var firstId = imported.mappings[0].recordId; var secondId = imported.mappings[1].recordId; + assert.equal((await service.readRecord({ databaseId: cases.id, recordId: firstId })).record.values.fld_parent0.id, secondId); + var corrected = await service.updateRecord({ databaseId: cases.id, recordId: firstId, expectedRevision: 1, values: { fld_notes00: "Human\n\tcorrection" }, operationId: op() }); assert.equal(corrected.record.revision, 2); + var exact = await other.importRecords(importArgs); assert.equal(exact.createdCount, 2); assert.equal(exact.mappings[0].recordId, firstId); + var fresh = await service.importRecords(Object.assign({}, importArgs, { operationId: op(), expectedDatabaseRevision: 3 })); assert.equal(fresh.createdCount, 0); assert.equal(fresh.databaseRevision, 3); + assert.equal((await service.readRecord({ databaseId: cases.id, recordId: firstId })).record.values.fld_notes00, "Human\n\tcorrection"); + var progress = await service.readImportProgress({ databaseId: cases.id, sourceNamespace: "source", limit: 1 }); assert.equal(progress.total, 2); assert.equal(progress.mappings[0].recordId, firstId); assert.equal(progress.nextOffset, 1); + var changed = JSON.parse(JSON.stringify(rows)); changed[1].sourcePayload.original = "changed"; + await rejection(function () { return service.importRecords(Object.assign({}, importArgs, { operationId: op(), expectedDatabaseRevision: 3, rows: changed })); }, "IMPORT_SOURCE_CONFLICT"); + await rejection(function () { return service.importRecords(Object.assign({}, importArgs, { rows: changed })); }, "IDEMPOTENCY_MISMATCH"); + await rejection(function () { return service.importRecords(Object.assign({}, importArgs, { operationId: op(), expectedDatabaseRevision: 2 })); }, "STALE"); + await rejection(function () { return service.importRecords(Object.assign({}, importArgs, { operationId: op(), expectedDatabaseRevision: 3, rows: [{ sourceKey: "new", sourcePayload: {}, values: { fld_title00: "New" } }, { sourceKey: "bad", sourcePayload: {}, values: { fld_title00: 3 } }] })); }, "INVALID"); + await rejection(function () { return service.importRecords(Object.assign({}, importArgs, { operationId: op(), expectedDatabaseRevision: 3, rows: [{ sourceKey: "outside", sourcePayload: {}, values: { fld_title00: "Outside", fld_parent0: { type: "record", databaseId: "db_outside0", id: "rec_outside" } } }] })); }, "FORBIDDEN_REFERENCE"); + assert.equal((await service.query({ databaseId: cases.id })).totals.count, 2); assert.equal((await service.read({ databaseId: cases.id })).database.revision, 3); + var archive = await service.archiveRecord({ databaseId: cases.id, recordId: secondId, expectedRevision: 1, operationId: op() }); assert.ok(archive.record.archivedAt); + assert.equal((await service.query({ databaseId: cases.id })).records[0].references[0].status, "archived"); assert.equal((await service.query({ databaseId: cases.id, archived: true })).totals.count, 1); + await service.recoverRecord({ databaseId: cases.id, recordId: secondId, expectedRevision: 2, operationId: op() }); assert.equal((await service.readRecord({ databaseId: cases.id, recordId: secondId })).history.length, 2); + var revision = (await service.read({ databaseId: cases.id })).database.revision; + var renamed = await service.rename({ databaseId: cases.id, expectedRevision: revision, name: "Renamed", operationId: op() }); assert.equal(renamed.database.name, "Renamed"); + var normal = await service.createRecord({ databaseId: cases.id, expectedDatabaseRevision: renamed.database.revision, values: { fld_title00: "Normal", fld_amount0: 12 }, operationId: op() }); assert.equal(normal.record.revision, 1); + revision = (await service.read({ databaseId: cases.id })).database.revision; + var view = { id: "view_amount0", name: "Amounts", columns: ["fld_title00", "fld_amount0"], filters: [{ fieldId: "fld_amount0", op: "gt", value: 4 }], sorts: [{ fieldId: "fld_amount0", direction: "desc" }] }; + var viewed = await service.saveView({ databaseId: cases.id, expectedRevision: revision, view: view, operationId: op() }); + var page = await service.query({ databaseId: cases.id, viewId: view.id, limit: 1 }); assert.equal(page.totals.count, 2); assert.equal(page.totals.sums.fld_amount0, 17); assert.equal(page.records[0].id, normal.record.id); + var nextPage = await service.query({ databaseId: cases.id, viewId: view.id, limit: 1, cursor: page.nextCursor }); assert.equal(nextPage.records[0].id, firstId); assert.equal(nextPage.nextCursor, null); + await rejection(function () { return service.updateSchema({ databaseId: cases.id, expectedRevision: viewed.database.revision, schema: viewed.database.schema.map(function (field) { return field.id === "fld_amount0" ? Object.assign({}, field, { type: "text" }) : field; }), operationId: op() }); }, "DESTRUCTIVE_SCHEMA"); + var schemaChanged = await service.updateSchema({ databaseId: cases.id, expectedRevision: viewed.database.revision, schema: viewed.database.schema.concat([{ id: "fld_extra00", name: "Extra", type: "boolean" }]), operationId: op() }); assert.equal(schemaChanged.database.schema.length, cases.schema.length + 1); + var competing = await Promise.allSettled([service.updateRecord({ databaseId: cases.id, recordId: firstId, expectedRevision: 2, values: { fld_extra00: true }, operationId: op() }), other.updateRecord({ databaseId: cases.id, recordId: firstId, expectedRevision: 2, values: { fld_extra00: false }, operationId: op() })]); + assert.equal(competing.filter(function (result) { return result.status === "fulfilled"; }).length, 1); assert.equal(competing.filter(function (result) { return result.status === "rejected" && result.reason.code === "STALE"; }).length, 1); + failProjection = true; var projected = await service.updateRecord({ databaseId: cases.id, recordId: firstId, expectedRevision: 3, values: { fld_notes00: "Canonical despite outage" }, operationId: op() }); assert.equal(projected.projectionPending, true); assert.match(projected.projectionError, /outage/); + assert.equal((await service.readRecord({ databaseId: cases.id, recordId: firstId })).record.values.fld_notes00, "Canonical despite outage"); + failProjection = false; assert.equal((await service.retryProjection({ databaseId: cases.id })).projectionPending, false); assert.ok(summaries.length > 0); + revision = (await service.read({ databaseId: cases.id })).database.revision; await service.archive({ databaseId: cases.id, expectedRevision: revision, operationId: op() }); + await rejection(function () { return service.query({ databaseId: cases.id }); }, "ARCHIVED"); assert.equal((await service.readImportProgress({ databaseId: cases.id, sourceNamespace: "source" })).total, 2); + var archived = (await service.list({ archived: true })).databases[0]; assert.equal(archived.id, cases.id); await service.recover({ databaseId: cases.id, expectedRevision: archived.revision, operationId: op() }); + assert.equal(service.listCached({}).databases.length, 2); + clock += require("../lib/knowledge-db-service").IDEMPOTENCY_WINDOW_MS + 1; + await rejection(function () { return service.importRecords(importArgs); }, "OPERATION_EXPIRED"); + var recoveredRevision = (await service.read({ databaseId: cases.id })).database.revision; + var resumed = await service.importRecords(Object.assign({}, importArgs, { operationId: op(), expectedDatabaseRevision: recoveredRevision })); + assert.equal(resumed.createdCount, 0); assert.equal(resumed.databaseRevision, recoveredRevision); assert.equal(resumed.mappings[0].recordId, firstId); + assert.equal((await service.readRecord({ databaseId: cases.id, recordId: firstId })).record.values.fld_notes00, "Canonical despite outage"); + }); +}); + +test("adapter validation rejects invalid injection and native-shaped CAS results clearly", async function () { + assert.throws(function () { createService({ storageAdapter: {} }); }, function (cause) { return cause.code === "INVALID_STORAGE_ADAPTER" && /contractVersion/.test(cause.message); }); + assert.throws(function () { createService({ storageAdapter: { contractVersion: 1 } }); }, /initialize must be a function/); + assert.throws(function () { createService({ storageFactory: 1 }); }, /storageFactory must be a synchronous function/); + assert.throws(function () { createService({ storageFactory: function () { return Promise.resolve(createMemoryStorage()); } }); }, /contractVersion/); + var bad = createMemoryStorage(); bad.diagnostics = true; assert.throws(function () { createService({ storageAdapter: bad }); }, /diagnostics must be a function/); + var adapter = createMemoryStorage(); var s = createService({ storageAdapter: adapter }); var made = await db(s, "Bad CAS"); adapter.replaceIfRevision = function () { return Promise.resolve({ numAffected: 1 }); }; + await rejection(function () { return s.rename({ databaseId: made.id, expectedRevision: 1, name: "Invalid", operationId: op() }); }, "INVALID_STORAGE_ADAPTER"); + var nonAsync = createMemoryStorage(); nonAsync.get = function () { return null; }; await rejection(function () { return createService({ storageAdapter: nonAsync }).read({ databaseId: "db_missing0" }); }, "INVALID_STORAGE_ADAPTER"); + var wrongIdentity = createMemoryStorage(); wrongIdentity.get = function () { return Promise.resolve({ _id: "db_other000", revision: 1 }); }; await rejection(function () { return createService({ storageAdapter: wrongIdentity }).read({ databaseId: "db_missing0" }); }, "INVALID_STORAGE_ADAPTER"); + var wrongList = createMemoryStorage(); wrongList.list = function () { return Promise.resolve({}); }; await rejection(function () { return createService({ storageAdapter: wrongList }).list({}); }, "INVALID_STORAGE_ADAPTER"); +}); + +test("factory injection and central production default seam retain application aggregate limits", async function () { + var options; var adapter = createMemoryStorage(); + var service = createService({ projectDir: "logical-scope", storageFactory: function (input) { options = input; return adapter; }, maxAggregateBytes: 4000 }); + assert.deepEqual(options, { projectDir: "logical-scope" }); var made = await db(service, "Bounded"); + await rejection(function () { return service.createRecord({ databaseId: made.id, expectedDatabaseRevision: 1, values: { fld_title00: "x".repeat(3900) }, operationId: op() }); }, "LIMIT"); assert.equal((await service.query({ databaseId: made.id })).totals.count, 0); + var originalFactory = factory.createDefaultKnowledgeDbStorage; + try { factory.createDefaultKnowledgeDbStorage = function () { return createMemoryStorage(); }; var selected = createService({}); assert.equal((await db(selected, "Central seam")).name, "Central seam"); assert.equal(await selected.readStorageStatus(), null); } + finally { factory.createDefaultKnowledgeDbStorage = originalFactory; } +}); + +test("alternate adapter imports and executes the domain module without loading NeDB or its adapter", function () { + var script = 'var assert=require("assert/strict");var Module=require("module");var original=Module._load;Module._load=function(name,parent,main){if(name.indexOf("nedb")!==-1)throw new Error("NeDB dependency was loaded");return original.call(this,name,parent,main);};var create=require(process.argv[1]).createKnowledgeDbService;var memory=require(process.argv[2]).createMemoryStorage;var s=create({storageAdapter:memory()});(async function(){assert.equal(await s.initialize(),undefined);var db=(await s.create({name:"Independent",operationId:"create_"+Date.now()+"_one",schema:[{id:"fld_title00",name:"Title",type:"title",required:true}]})).database;var imported=await s.importRecords({databaseId:db.id,expectedDatabaseRevision:1,operationId:"import_"+Date.now()+"_one",sourceNamespace:"independent",rows:[{sourceKey:"one",sourcePayload:{original:true},values:{fld_title00:"One"}}]});assert.equal(imported.createdCount,1);assert.equal((await s.query({databaseId:db.id})).totals.count,1);assert.equal(await s.readStorageStatus(),null);process.stdout.write("independent");})().catch(function(e){console.error(e);process.exit(1);});'; + assert.equal(child.execFileSync(process.execPath, ["-e", script, path.resolve(__dirname, "../lib/knowledge-db-service.js"), path.resolve(__dirname, "fixtures/knowledge-db-memory-adapter.cjs")], { encoding: "utf8" }), "independent"); +}); diff --git a/test/mate-knowledge-import-tools.test.js b/test/mate-knowledge-import-tools.test.js new file mode 100644 index 00000000..87036b9c --- /dev/null +++ b/test/mate-knowledge-import-tools.test.js @@ -0,0 +1,195 @@ +var test = require("node:test"); +var assert = require("node:assert/strict"); +var fs = require("fs"); +var os = require("os"); +var path = require("path"); +var zod = require("zod"); + +var attachService = require("../lib/mate-knowledge-service").attachMateKnowledgeService; +var attachProjectMateKnowledge = require("../lib/project-mate-knowledge").attachProjectMateKnowledge; +var createKnowledgeDbService = require("../lib/knowledge-db-service").createKnowledgeDbService; +var importLimits = require("../lib/knowledge-db-import"); + +var sequence = 0; +function operation(prefix) { sequence++; return prefix + "_" + Date.now() + "_" + sequence; } +function call(attached, session, name, args) { + var defs = attached.getToolDefs(session); + for (var i = 0; i < defs.length; i++) if (defs[i].name === name) return defs[i].handler(args || {}); + throw new Error("Tool not advertised: " + name); +} +function validate(attached, session, name, args) { + var defs = attached.getToolDefs(session); + for (var i = 0; i < defs.length; i++) if (defs[i].name === name) return zod.object(defs[i].inputSchema).parse(args || {}); + throw new Error("Tool not advertised: " + name); +} +function callParsed(attached, session, name, args) { + return call(attached, session, name, validate(attached, session, name, args)); +} +function definition(attached, session, name) { + var defs = attached.getToolDefs(session); + for (var i = 0; i < defs.length; i++) if (defs[i].name === name) return defs[i]; + throw new Error("Tool not advertised: " + name); +} +async function json(result) { + var value = await result; + if (value.isError) throw new Error(value.content[0].text); + return JSON.parse(value.content[0].text); +} +async function error(result, pattern) { + var value = await result; + assert.equal(value.isError, true); + if (pattern) assert.match(value.content[0].text, pattern); + return value; +} + +function fixture(t) { + var root = fs.mkdtempSync(path.join(os.tmpdir(), "clay-knowledge-import-tools-")); + var mateDir = path.join(root, "mates", "alice", "mate_import"); + fs.mkdirSync(path.join(mateDir, "knowledge"), { recursive: true }); + t.after(function () { fs.rmSync(root, { recursive: true, force: true }); }); + var session = { localId: 1, cliSessionId: "import", ownerId: "alice" }; + var foreign = { localId: 2, cliSessionId: "foreign", ownerId: "bob" }; + var sessions = new Map([[session.localId, session]]); + var databaseService; + function newDatabaseService() { databaseService = createKnowledgeDbService({ projectDir: mateDir, authorizeTextReference: function () { return false; } }); } + newDatabaseService(); + var project = { + getStatus: function () { return { isMate: true, mateId: "mate_import", projectOwnerId: "alice", path: mateDir }; }, + getSessionManager: function () { return { sessions: sessions }; }, + getKnowledgeDbService: function () { return databaseService; }, + }; + var projects = new Map([["mate-import", project]]); + var mate = { id: "mate_import", name: "Import", createdBy: "alice", dir: mateDir }; + function service() { + return attachService({ + baseDir: path.join(root, "record-store"), getProjects: function () { return projects; }, + isMultiUser: function () { return true; }, + resolveMate: function (userId, mateId) { return userId === "alice" && mateId === mate.id ? mate : null; }, + listMates: function (userId) { return userId === "alice" ? [mate] : []; }, + }); + } + var boundService = service(); + function attach() { return attachProjectMateKnowledge({ service: boundService, sm: project.getSessionManager(), projectSlug: "mate-import", isMate: true, mateId: mate.id, getProjectOwnerId: function () { return "alice"; } }); } + return { + root: root, mateDir: mateDir, session: session, foreign: foreign, project: project, attach: attach, + rebind: function () { newDatabaseService(); boundService = service(); return attach(); }, + }; +} + +test("ordinary Mate import tools register across native MCP bridge and preserve source mappings", async function (t) { + var f = fixture(t); var tools = f.attach(); + var toolDefinitions = tools.getToolDefs(f.session); + var importDefinition = toolDefinitions.filter(function (tool) { return tool.name === "import_database_records"; })[0]; + var definitions = toolDefinitions.map(function (tool) { return tool.name; }); + assert.ok(definitions.indexOf("import_database_records") !== -1); + assert.ok(definitions.indexOf("read_database_import_progress") !== -1); + assert.equal(importDefinition.inputSchema.rows.type, "array"); + var bridgeDefinitions = tools.getBridgeTools(f.session, function (schema) { return schema; }); + var bridge = bridgeDefinitions.map(function (tool) { return tool.name; }); + assert.ok(bridge.indexOf("import_database_records") !== -1); + assert.equal(bridgeDefinitions.filter(function (tool) { return tool.name === "import_database_records"; })[0].inputSchema.rows.type, "array"); + var server = tools.createMcpServer({ createToolServer: function (definition) { return definition; } }, f.session); + assert.ok(server.tools.some(function (tool) { return tool.name === "read_database_import_progress"; })); + + var importJsonSchema = zod.toJSONSchema(zod.object(importDefinition.inputSchema)); + assert.equal(importJsonSchema.properties.rows.type, "array"); + assert.ok(importJsonSchema.properties.rows.items.required.indexOf("sourcePayload") !== -1); + assert.equal(zod.toJSONSchema(zod.object(definition(tools, f.session, "create_database").inputSchema)).properties.schema.type, "array"); + assert.equal(zod.toJSONSchema(zod.object(definition(tools, f.session, "query_database").inputSchema)).properties.filters.type, "array"); + assert.equal(zod.toJSONSchema(zod.object(definition(tools, f.session, "query_database").inputSchema)).properties.sorts.type, "array"); + assert.equal(zod.toJSONSchema(zod.object(definition(tools, f.session, "create_database_record").inputSchema)).properties.values.type, "object"); + assert.equal(zod.toJSONSchema(zod.object(definition(tools, f.session, "save_table_view").inputSchema)).properties.view.type, "object"); + + var createDatabaseArgs = { + name: "Imported vocabulary", operationId: operation("db"), schema: [ + { id: "fld_title", name: "Word", type: "title", required: true }, + { id: "fld_notes", name: "Notes", type: "text" }, + { id: "fld_related", name: "Related", type: "reference", target: "record" }, + ], + }; + assert.doesNotThrow(function () { validate(tools, f.session, "create_database", createDatabaseArgs); }); + var db = (await json(callParsed(tools, f.session, "create_database", createDatabaseArgs))).database; + var rows = [ + { sourceKey: "one", sourcePayload: { source: "one", original: "alpha" }, values: { fld_title: "Alpha", fld_notes: "Original alpha" } }, + { sourceKey: "two", sourcePayload: { source: "two", original: "beta" }, values: { fld_title: "Beta", fld_notes: "Original beta", fld_related: { type: "record", sourceNamespace: "vocabulary", sourceKey: "one" } } }, + ]; + var request = { databaseId: db.id, expectedDatabaseRevision: db.revision, operationId: operation("import"), sourceNamespace: "vocabulary", rows: rows }; + assert.doesNotThrow(function () { validate(tools, f.session, "import_database_records", request); }); + var imported = await json(callParsed(tools, f.session, "import_database_records", request)); + assert.equal(imported.createdCount, 2); + assert.equal(imported.replayedCount, 0); + assert.equal(imported.mappings[0].created, true); + assert.equal(imported.projectionPending, false); + var record = await json(call(tools, f.session, "read_database_record", { databaseId: db.id, recordId: imported.mappings[1].recordId })); + assert.equal(record.record.values.fld_related.id, imported.mappings[0].recordId); + assert.equal(record.record.createdBy.userId, "alice"); + var correctionArgs = { databaseId: db.id, recordId: record.record.id, values: { fld_notes: "Human correction\nkept" }, expectedRevision: record.record.revision, operationId: operation("correction") }; + assert.doesNotThrow(function () { validate(tools, f.session, "update_database_record", correctionArgs); }); + await json(callParsed(tools, f.session, "update_database_record", correctionArgs)); + var currentDatabase = (await json(call(tools, f.session, "read_database", { databaseId: db.id }))).database; + var expandedSchema = currentDatabase.schema.concat([{ id: "fld_extra", name: "Extra", type: "text" }]); + var schemaArgs = { databaseId: db.id, schema: expandedSchema, expectedRevision: currentDatabase.revision, operationId: operation("schema") }; + assert.doesNotThrow(function () { validate(tools, f.session, "update_database_schema", schemaArgs); }); + var expanded = (await json(callParsed(tools, f.session, "update_database_schema", schemaArgs))).database; + var viewArgs = { databaseId: db.id, view: { id: "view_imported", name: "Imported notes", columns: expanded.schema.map(function (field) { return field.id; }), filters: [{ fieldId: "fld_notes", op: "contains", value: "Original" }], sorts: [{ fieldId: "fld_title", direction: "asc" }] }, expectedRevision: expanded.revision, operationId: operation("view") }; + assert.doesNotThrow(function () { validate(tools, f.session, "save_table_view", viewArgs); }); + await json(callParsed(tools, f.session, "save_table_view", viewArgs)); + var progress = await json(call(tools, f.session, "read_database_import_progress", { databaseId: db.id, sourceNamespace: "vocabulary", limit: 1 })); + assert.equal(progress.total, 2); + assert.equal(progress.mappings.length, 1); + assert.equal(progress.nextOffset, 1); + + var exact = await json(callParsed(tools, f.session, "import_database_records", request)); + assert.equal(exact.createdCount, 2); + assert.equal(exact.databaseRevision, imported.databaseRevision); + var latestDatabase = (await json(call(tools, f.session, "read_database", { databaseId: db.id }))).database; + var freshReplay = Object.assign({}, request, { expectedDatabaseRevision: latestDatabase.revision, operationId: operation("fresh-replay") }); + var replay = await json(callParsed(tools, f.session, "import_database_records", freshReplay)); + assert.equal(replay.createdCount, 0); + assert.equal(replay.replayedCount, 2); + assert.equal(replay.databaseRevision, latestDatabase.revision); + + var queryArgs = { databaseId: db.id, viewId: "view_imported", filters: [{ fieldId: "fld_notes", op: "contains", value: "Original" }], sorts: [{ fieldId: "fld_title", direction: "asc" }], limit: 20 }; + assert.doesNotThrow(function () { validate(tools, f.session, "query_database", queryArgs); }); + var queried = await json(callParsed(tools, f.session, "query_database", queryArgs)); + assert.equal(queried.totals.count, 1); + var changed = Object.assign({}, freshReplay, { operationId: operation("changed-source"), rows: [{ sourceKey: "one", sourcePayload: { source: "one", changed: true }, values: { fld_title: "Changed" } }] }); + await error(call(tools, f.session, "import_database_records", changed), /already imported with a different payload|conflict/i); + var stale = Object.assign({}, request, { operationId: operation("stale"), rows: [{ sourceKey: "three", sourcePayload: { source: "three" }, values: { fld_title: "Three" } }] }); + await error(call(tools, f.session, "import_database_records", stale), /revision|changed/i); + + var invalidRows = []; + for (var i = 0; i < importLimits.MAX_BATCH_ROWS + 1; i++) invalidRows.push({ sourceKey: "overflow-" + i, sourcePayload: { i: i }, values: { fld_title: "Overflow " + i } }); + await error(call(tools, f.session, "import_database_records", Object.assign({}, freshReplay, { operationId: operation("overflow"), rows: invalidRows })), /100|batch|limit/i); + await error(call(tools, f.session, "read_database_import_progress", { databaseId: db.id, sourceNamespace: "vocabulary", limit: 101 }), /limit|between/i); + + var freshTools = f.rebind(); + var freshProgress = await json(call(freshTools, f.session, "read_database_import_progress", { databaseId: db.id, sourceNamespace: "vocabulary" })); + assert.equal(freshProgress.total, 2); + assert.deepEqual(freshTools.getToolDefs(f.foreign), []); +}); + +test("raw forged actors cannot override server attribution", async function (t) { + var f = fixture(t); var tools = f.attach(); + var db = (await json(callParsed(tools, f.session, "create_database", { name: "Attribution", operationId: operation("actor-db"), schema: [{ id: "fld_title", name: "Title", type: "title" }] }))).database; + var rawRequest = { databaseId: db.id, expectedDatabaseRevision: db.revision, operationId: operation("forged-actor"), sourceNamespace: "actor", rows: [{ sourceKey: "one", sourcePayload: { original: "one" }, values: { fld_title: "One" } }], actor: { type: "user", userId: "bob" } }; + var imported = await json(call(tools, f.session, "import_database_records", rawRequest)); + var record = await json(call(tools, f.session, "read_database_record", { databaseId: db.id, recordId: imported.mappings[0].recordId })); + assert.equal(record.record.createdBy.userId, "alice"); +}); + +test("captured import handlers reject live owner mutation and session removal", async function (t) { + var f = fixture(t); var tools = f.attach(); + var db = (await json(callParsed(tools, f.session, "create_database", { name: "Guarded", operationId: operation("guarded-db"), schema: [{ id: "fld_title", name: "Title", type: "title" }] }))).database; + var captured = definition(tools, f.session, "import_database_records"); + var request = { databaseId: db.id, expectedDatabaseRevision: db.revision, operationId: operation("captured"), sourceNamespace: "guarded", rows: [{ sourceKey: "one", sourcePayload: { original: "one" }, values: { fld_title: "One" } }] }; + f.session.ownerId = "bob"; + await error(captured.handler(request), /owner|authorized|session/i); + f.session.ownerId = "alice"; + f.project.getSessionManager().sessions.delete(f.session.localId); + await error(captured.handler(request), /session|authorized|owner/i); + f.project.getSessionManager().sessions.set(f.session.localId, f.session); + var query = await json(callParsed(tools, f.session, "query_database", { databaseId: db.id, limit: 20 })); + assert.equal(query.totals.count, 0); + assert.deepEqual(tools.getToolDefs(f.foreign), []); +}); diff --git a/test/mate-knowledge-tools.test.js b/test/mate-knowledge-tools.test.js index acddcac7..b9d885ae 100644 --- a/test/mate-knowledge-tools.test.js +++ b/test/mate-knowledge-tools.test.js @@ -6,13 +6,14 @@ var path = require("path"); var attachService = require("../lib/mate-knowledge-service").attachMateKnowledgeService; var attachProjectMateKnowledge = require("../lib/project-mate-knowledge").attachProjectMateKnowledge; +var attachKnowledge = require("../lib/project-knowledge").attachKnowledge; var knowledgeMcp = require("../lib/mate-knowledge-mcp-server"); var knowledgeSearch = require("../lib/knowledge-search"); var knowledgeImport = require("../lib/knowledge-import"); var mateSync = require("../lib/mate-knowledge-sync"); var createKnowledgeDbService = require("../lib/knowledge-db-service").createKnowledgeDbService; -var MATE_TOOLS = ["list_knowledge", "search_knowledge", "read_knowledge", "list_databases", "create_database", "read_database", "rename_database", "archive_database", "recover_database", "update_database_schema", "save_table_view", "query_database", "create_database_record", "read_database_record", "update_database_record", "archive_database_record", "recover_database_record", "retry_database_sync"]; +var MATE_TOOLS = ["list_knowledge", "search_knowledge", "read_knowledge", "read_knowledge_document", "create_knowledge_document", "update_knowledge_document", "list_knowledge_document_history", "restore_knowledge_document", "list_databases", "create_database", "read_database", "rename_database", "archive_database", "recover_database", "update_database_schema", "save_table_view", "query_database", "create_database_record", "read_database_record", "update_database_record", "archive_database_record", "recover_database_record", "retry_database_sync", "import_database_records", "read_database_import_progress"]; var CLAY_TOOLS = ["list_mate_knowledge", "search_mate_knowledge", "read_mate_knowledge"]; var databaseOperationSequence = 0; function databaseOperation(prefix) { databaseOperationSequence++; return prefix + "_" + Date.now() + "_" + databaseOperationSequence; } @@ -245,6 +246,146 @@ test("an ordinary Mate cannot read another Mate's record by its opaque ref", asy assert.equal(badRef.isError, true); }); +test("session-bound text Knowledge tools guard revisions, history, sync, paths, and Mate ownership", async function (t) { + var w = workspace(); + var captured = w.arch.getToolDefs(w.sessions.arch).filter(function (tool) { return tool.name === "read_knowledge_document"; })[0]; + var unknownId = await call(w.arch, w.sessions.arch, "read_knowledge_document", { documentId: "missing-document-id", name: "architecture.md" }); + assert.equal(unknownId.isError, true); + var created = await json(call(w.arch, w.sessions.arch, "create_knowledge_document", { + name: "areas/agent-notes.md", content: "# Agent notes\nInitial narrative.\n", + })); + assert.match(created.documentId, /^[a-f0-9-]{20,50}$/); + assert.ok(created.revision); + + var read = await json(call(w.arch, w.sessions.arch, "read_knowledge_document", { documentId: created.documentId })); + assert.equal(read.content, "# Agent notes\nInitial narrative.\n"); + assert.equal(read.name, "areas/agent-notes.md"); + assert.equal(read.revision, created.revision); + + var updated = await json(call(w.arch, w.sessions.arch, "update_knowledge_document", { + documentId: created.documentId, content: "# Agent notes\nUpdated narrative with searchable marker.\n", expectedRevision: created.revision, + })); + assert.notEqual(updated.revision, created.revision); + var stale = await call(w.arch, w.sessions.arch, "update_knowledge_document", { + documentId: created.documentId, content: "stale", expectedRevision: created.revision, + }); + assert.equal(stale.isError, true); + assert.match(stale.content[0].text, /changed since it was opened|revision/i); + + var history = await json(call(w.arch, w.sessions.arch, "list_knowledge_document_history", { documentId: created.documentId })); + assert.equal(history.versions.length, 1); + var restored = await json(call(w.arch, w.sessions.arch, "restore_knowledge_document", { + documentId: created.documentId, version: history.versions[0].version, expectedRevision: updated.revision, + })); + assert.ok(restored.revision); + var restoredRead = await json(call(w.arch, w.sessions.arch, "read_knowledge_document", { name: "areas/agent-notes.md" })); + assert.equal(restoredRead.content, "# Agent notes\nInitial narrative.\n"); + + var found = await json(call(w.arch, w.sessions.arch, "search_knowledge", { query: "searchable marker" })); + assert.equal(found.total, 0, "restore removes the revised text from the search projection"); + var updatedAgain = await json(call(w.arch, w.sessions.arch, "update_knowledge_document", { + name: "areas/agent-notes.md", content: "# Agent notes\nSearchable marker survives restart.\n", expectedRevision: restoredRead.revision, + })); + assert.ok(updatedAgain.indexSyncPending === false); + var searched = await json(call(w.arch, w.sessions.arch, "search_knowledge", { query: "survives restart" })); + assert.equal(searched.total, 1); + + var restartedService = attachService({ + baseDir: w.baseDir, + getProjects: function () { return w.projects; }, + isMultiUser: function () { return true; }, + resolveMate: function (userId, mateId) { + var mate = w.registry[mateId]; + return mate && mate.createdBy === userId ? mate : null; + }, + listMates: function (userId) { + return Object.keys(w.registry).map(function (key) { return w.registry[key]; }).filter(function (mate) { return mate.createdBy === userId; }); + }, + }); + var restarted = attachProjectMateKnowledge({ + service: restartedService, sm: w.projects.get("mate-arch").getSessionManager(), projectSlug: "mate-arch", + getProjectOwnerId: function () { return "alice"; }, isMate: true, mateId: "mate_arch", + }); + var restartRead = await json(call(restarted, w.sessions.arch, "read_knowledge_document", { documentId: created.documentId })); + assert.equal(restartRead.content, "# Agent notes\nSearchable marker survives restart.\n"); + var restartSearch = await json(call(restarted, w.sessions.arch, "search_knowledge", { query: "survives restart" })); + assert.equal(restartSearch.total, 1); + + var foreign = await call(w.scribe, w.sessions.scribe, "read_knowledge_document", { documentId: created.documentId }); + assert.equal(foreign.isError, true); + var bob = await call(w.bob, w.sessions.bob, "read_knowledge_document", { documentId: created.documentId }); + assert.equal(bob.isError, true); + assert.deepEqual(w.driver.getToolDefs(w.sessions.driver), []); + + w.sessions.arch.ownerId = "bob"; + var mutableOwner = await captured.handler({ documentId: created.documentId }); + assert.equal(mutableOwner.isError, true, "a captured handler rechecks the live session owner"); + w.sessions.arch.ownerId = "alice"; + + var outside = path.join(w.home, "outside"); + fs.mkdirSync(outside); + fs.symlinkSync(outside, path.join(w.dirs.arch, "knowledge", "linked")); + var linked = await call(w.arch, w.sessions.arch, "create_knowledge_document", { name: "linked/escape.md", content: "nope" }); + assert.equal(linked.isError, true); + assert.equal(fs.existsSync(path.join(outside, "escape.md")), false); +}); + +test("text tools and the Knowledge workbench share ids, history, conflicts, and restart state", async function (t) { + var w = workspace(); + var sent = []; + var mates = { + buildMateCtx: function () { return {}; }, + isPromoted: function () { return false; }, + getCommonKnowledgeForMate: function () { return []; }, + readCommonKnowledgeFile: function () { throw new Error("no shared file"); }, + }; + function attachUi() { + return attachKnowledge({ + cwd: w.dirs.arch, isMate: true, knowledgeBaseDir: w.baseDir, + sendTo: function (_ws, message) { sent.push(message); }, + matesModule: mates, getProjectOwnerId: function () { return "alice"; }, + }); + } + var ui = attachUi(); + function uiMessage(message) { + sent.length = 0; + ui.handleKnowledgeMessage({ _clayUser: { id: "alice" } }, message); + return sent.filter(function (entry) { return entry.type; })[0]; + } + var uiCreated = uiMessage({ type: "knowledge_save", requestId: "ui-create", name: "interop.md", content: "UI version" }); + assert.ok(uiCreated.revision); + var agentRead = await json(call(w.arch, w.sessions.arch, "read_knowledge_document", { name: "interop.md" })); + assert.equal(agentRead.documentId, uiCreated.id); + assert.equal(agentRead.revision, uiCreated.revision); + + var agentUpdate = await json(call(w.arch, w.sessions.arch, "update_knowledge_document", { + documentId: agentRead.documentId, content: "Agent version", expectedRevision: agentRead.revision, + })); + var uiStale = uiMessage({ type: "knowledge_save", requestId: "ui-stale", name: "interop.md", content: "lost", expectedRevision: uiCreated.revision }); + assert.equal(uiStale.type, "knowledge_error"); + assert.equal(uiStale.code, "STALE"); + assert.equal(fs.readFileSync(path.join(w.dirs.arch, "knowledge", "interop.md"), "utf8"), "Agent version"); + + var uiRead = uiMessage({ type: "knowledge_read", requestId: "ui-read", documentId: agentUpdate.documentId, name: "interop.md" }); + assert.equal(uiRead.content, "Agent version"); + var uiHistory = uiMessage({ type: "knowledge_history", requestId: "ui-history", documentId: agentUpdate.documentId }); + assert.ok(uiHistory.versions.length >= 1); + ui = attachUi(); + var afterRestart = uiMessage({ type: "knowledge_read", requestId: "ui-restart", documentId: agentUpdate.documentId, name: "interop.md" }); + assert.equal(afterRestart.revision, agentUpdate.revision); +}); + +test("text Knowledge tools surface a committed write when projection synchronization fails", async function (t) { + var w = workspace(); + var original = mateSync.syncMateSource; + mateSync.syncMateSource = function () { throw new Error("fixture projection unavailable"); }; + t.after(function () { mateSync.syncMateSource = original; }); + var saved = await json(call(w.arch, w.sessions.arch, "create_knowledge_document", { name: "offline-agent.md", content: "Canonical text" })); + assert.equal(saved.indexSyncPending, true); + assert.match(saved.syncError, /projection unavailable/); + assert.equal(fs.readFileSync(path.join(w.dirs.arch, "knowledge", "offline-agent.md"), "utf8"), "Canonical text"); +}); + test("session-bound native Knowledge tools create and operate only the owning Mate database", async function () { var w = workspace(); var createOperation = databaseOperation("create-cases"); diff --git a/test/mate-knowledge-workflows.test.js b/test/mate-knowledge-workflows.test.js new file mode 100644 index 00000000..f178c80a --- /dev/null +++ b/test/mate-knowledge-workflows.test.js @@ -0,0 +1,176 @@ +var test = require("node:test"); +var assert = require("node:assert/strict"); +var fs = require("fs"); +var os = require("os"); +var path = require("path"); + +var attachService = require("../lib/mate-knowledge-service").attachMateKnowledgeService; +var attachProjectMateKnowledge = require("../lib/project-mate-knowledge").attachProjectMateKnowledge; +var mateSync = require("../lib/mate-knowledge-sync"); +var createKnowledgeDbService = require("../lib/knowledge-db-service").createKnowledgeDbService; + +var operationSequence = 0; +function operation(prefix) { operationSequence++; return prefix + "_" + Date.now() + "_" + operationSequence; } +function call(attached, session, name, args) { + var defs = attached.getToolDefs(session); + for (var i = 0; i < defs.length; i++) if (defs[i].name === name) return defs[i].handler(args || {}); + throw new Error("Tool not advertised: " + name); +} +async function json(result) { + var value = await result; + if (value.isError) throw new Error(value.content[0].text); + return JSON.parse(value.content[0].text); +} + +function fixture(t) { + var root = fs.mkdtempSync(path.join(os.tmpdir(), "clay-knowledge-workflow-")); + var mateDir = path.join(root, "mates", "alice", "mate_workflow"); + var baseDir = path.join(root, "record-store"); + fs.mkdirSync(path.join(mateDir, "knowledge"), { recursive: true }); + t.after(function () { fs.rmSync(root, { recursive: true, force: true }); }); + mateSync.reconcileMate({ mateDir: mateDir, baseDir: baseDir, actor: { type: "test" } }); + + var session = { localId: 1, cliSessionId: "workflow", ownerId: "alice" }; + var outsider = { localId: 2, cliSessionId: "outsider", ownerId: "bob" }; + var sessions = new Map([[session.localId, session]]); + function newDatabaseService() { return createKnowledgeDbService({ projectDir: mateDir, authorizeTextReference: function () { return false; } }); } + var databaseService = newDatabaseService(); + var project = { + getStatus: function () { return { isMate: true, mateId: "mate_workflow", projectOwnerId: "alice", path: mateDir }; }, + getSessionManager: function () { return { sessions: sessions }; }, + getKnowledgeDbService: function () { return databaseService; }, + }; + var projects = new Map([["mate-workflow", project]]); + var registry = { id: "mate_workflow", name: "Workflow", createdBy: "alice", dir: mateDir }; + function buildService() { + return attachService({ + baseDir: baseDir, + getProjects: function () { return projects; }, + isMultiUser: function () { return true; }, + resolveMate: function (userId, mateId) { return mateId === registry.id && userId === registry.createdBy ? registry : null; }, + listMates: function (userId) { return userId === "alice" ? [registry] : []; }, + }); + } + function attach(service) { + return attachProjectMateKnowledge({ + service: service, sm: project.getSessionManager(), projectSlug: "mate-workflow", isMate: true, + mateId: registry.id, getProjectOwnerId: function () { return "alice"; }, + }); + } + var service = buildService(); + return { root: root, mateDir: mateDir, baseDir: baseDir, session: session, outsider: outsider, project: project, attach: function () { return attach(service); }, restart: function () { databaseService = newDatabaseService(); service = buildService(); return attach(service); } }; +} + +test("session-bound structured workflows use schema, references, typed filters, and scoped totals", async function (t) { + var f = fixture(t); + var tools = f.attach(); + var contacts = (await json(call(tools, f.session, "create_database", { + name: "Contacts", operationId: operation("contacts"), schema: [ + { id: "fld_contact_title", name: "Name", type: "title", required: true }, + { id: "fld_cadence", name: "Cadence", type: "text" }, + { id: "fld_followup", name: "Next follow-up", type: "date" }, + ], + }))).database; + var contactTitle = contacts.schema[0].id; + var contact = await json(call(tools, f.session, "create_database_record", { + databaseId: contacts.id, expectedDatabaseRevision: contacts.revision, operationId: operation("contact"), + values: (function () { var values = {}; values[contactTitle] = "Alex Example"; values.fld_cadence = "weekly"; values.fld_followup = "2026-10-20"; return values; })(), + })); + var transactions = (await json(call(tools, f.session, "create_database", { + name: "Transactions", operationId: operation("transactions"), schema: [ + { id: "fld_tx_title", name: "Description", type: "title", required: true }, + { id: "fld_category", name: "Category", type: "text" }, + { id: "fld_amount", name: "Amount", type: "number" }, + { id: "fld_spender", name: "Spender", type: "reference", target: "record", targetDatabaseId: contacts.id }, + ], + }))).database; + var txTitle = transactions.schema[0].id; + async function addTransaction(title, category, amount, label) { + var current = (await json(call(tools, f.session, "read_database", { databaseId: transactions.id }))).database; + var values = {}; values[txTitle] = title; values.fld_category = category; values.fld_amount = amount; + values.fld_spender = { type: "record", databaseId: contacts.id, id: contact.record.id }; + return json(call(tools, f.session, "create_database_record", { databaseId: transactions.id, expectedDatabaseRevision: current.revision, operationId: operation(label), values: values })); + } + await addTransaction("Lunch", "food", 18, "lunch"); + await addTransaction("Train", "travel", 42, "train"); + await addTransaction("Hotel", "travel", 120, "hotel"); + var schemaRead = await json(call(tools, f.session, "read_database", { databaseId: transactions.id })); + var query = await json(call(tools, f.session, "query_database", { + databaseId: transactions.id, + filters: [{ fieldId: "fld_category", op: "eq", value: "travel" }], + sorts: [{ fieldId: "fld_amount", direction: "desc" }], limit: 1, + })); + assert.equal(schemaRead.database.schema.filter(function (field) { return field.id === "fld_amount"; })[0].type, "number"); + assert.equal(query.records.length, 1); + assert.equal(query.totals.count, 2); + assert.equal(query.totals.sums.fld_amount, 162); + assert.equal(query.records[0].values.fld_amount, 120); + assert.equal(query.records[0].references[0].status, "available"); + + var followUp = await json(call(tools, f.session, "query_database", { + databaseId: contacts.id, + filters: [ + { fieldId: "fld_cadence", op: "eq", value: "weekly" }, + { fieldId: "fld_followup", op: "lte", value: "2026-10-21" }, + ], limit: 20, + })); + assert.equal(followUp.totals.count, 1, "the caller supplied the cadence/date cutoff"); + assert.equal(followUp.records[0].id, contact.record.id); + var freshTools = f.restart(); + var freshQuery = await json(call(freshTools, f.session, "query_database", { + databaseId: transactions.id, filters: [{ fieldId: "fld_category", op: "eq", value: "travel" }], limit: 20, + })); + assert.equal(freshQuery.totals.count, 2, "fresh service rebind retains structured records and totals"); + assert.deepEqual(f.attach().getToolDefs(f.outsider), []); +}); + +test("narrative and correction workflows preserve links, revisions, history, rebind state, and original text", async function (t) { + var f = fixture(t); + var tools = f.attach(); + var created = await json(call(tools, f.session, "create_knowledge_document", { + name: "daily/2026-10-13.md", content: "# Daily capture\nSee [[projects/plan]].\nOriginal English word: accomodate.\n", + })); + var edited = await json(call(tools, f.session, "update_knowledge_document", { + documentId: created.documentId, expectedRevision: created.revision, + content: "# Daily capture\nSee [[projects/plan]].\nOriginal English word: accommodate.\n", + })); + var stale = await call(tools, f.session, "update_knowledge_document", { + documentId: created.documentId, expectedRevision: created.revision, content: "lost correction", + }); + assert.equal(stale.isError, true); + var current = await json(call(tools, f.session, "read_knowledge_document", { documentId: created.documentId })); + assert.match(current.content, /\[\[projects\/plan\]\]/); + assert.match(current.content, /accommodate/); + var history = await json(call(tools, f.session, "list_knowledge_document_history", { documentId: created.documentId })); + assert.ok(history.versions.length >= 1); + + var restored = await json(call(tools, f.session, "restore_knowledge_document", { + documentId: created.documentId, version: history.versions[0].version, expectedRevision: edited.revision, + })); + assert.ok(restored.revision); + var restarted = f.restart(); + var afterRestart = await json(call(restarted, f.session, "read_knowledge_document", { documentId: created.documentId })); + assert.match(afterRestart.content, /Original English word: accomodate/); + assert.match(afterRestart.content, /\[\[projects\/plan\]\]/); + assert.deepEqual(restarted.getToolDefs(f.outsider), [], "an outsider receives no Knowledge handler"); +}); + +test("correction records retain original English text while query and update use stable ids", async function (t) { + var f = fixture(t); + var tools = f.attach(); + var made = (await json(call(tools, f.session, "create_database", { + name: "Corrections", operationId: operation("corrections"), schema: [ + { id: "fld_word_title", name: "Word", type: "title", required: true }, + { id: "fld_original", name: "Original", type: "text" }, + { id: "fld_correction", name: "Correction", type: "text" }, + ], + }))).database; + var values = {}; values.fld_word_title = "accomodate"; values.fld_original = "The original English spelling was accomodate."; values.fld_correction = "accommodate"; + var created = await json(call(tools, f.session, "create_database_record", { databaseId: made.id, expectedDatabaseRevision: made.revision, operationId: operation("word"), values: values })); + var found = await json(call(tools, f.session, "query_database", { databaseId: made.id, filters: [{ fieldId: "fld_word_title", op: "eq", value: "accomodate" }] })); + assert.equal(found.totals.count, 1); + var corrected = await json(call(tools, f.session, "update_database_record", { databaseId: made.id, recordId: created.record.id, expectedRevision: created.record.revision, operationId: operation("correct"), values: { fld_correction: "accommodate (corrected)" } })); + var read = await json(call(tools, f.session, "read_database_record", { databaseId: made.id, recordId: corrected.record.id })); + assert.equal(read.record.values.fld_original, "The original English spelling was accomodate."); + assert.equal(read.record.values.fld_correction, "accommodate (corrected)"); +});