From 75b1ac12df1e6b8cdf2170ccd7b6f51169024d0f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 10:31:58 +0100 Subject: [PATCH 01/76] feat(onboarding): enable lean mode for local models (#110596) --- docs/.generated/config-baseline.sha256 | 6 +- docs/concepts/experimental-features.md | 6 +- src/commands/onboard-guided.custodian.test.ts | 22 ++++ src/config/local-model-lean-auto.test.ts | 91 ++++++++++++++++ src/config/local-model-lean-auto.ts | 101 ++++++++++++++++++ src/config/schema.help.core.ts | 2 + src/config/schema.labels.ts | 1 + src/config/types.openclaw.ts | 2 + src/config/zod-schema.root-shape.ts | 1 + src/system-agent/setup-inference.test.ts | 53 ++++++++- src/system-agent/setup-inference.ts | 31 +++++- 11 files changed, 305 insertions(+), 11 deletions(-) create mode 100644 src/config/local-model-lean-auto.test.ts create mode 100644 src/config/local-model-lean-auto.ts diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index 753bc17ead0..85012a3c2af 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -cc6b32ad0a77ec983c3b97fadee565f93e0c9437845335a9dd55ea83e60a366d config-baseline.json -17d536dc4bb6a97a18120cb4ed1884a2f4e250a881277cee28dfe26aaeeb005c config-baseline.core.json +53e7eea9a0193d94f5afe8e774456ae71cd3eddf99e5224602fcf13bb845051b config-baseline.json +97c9ec219366ad2fab4d83d5b05eb3ae3df71db68747e7b31cc697a76676c620 config-baseline.core.json f75efa4f3fa4ac7c2e7dc6580a1dd1efaa8e2e13ec7df9c21838c8ebc44a3dca config-baseline.channel.json -ddde5320a53d2cb2bb41f4624ae09a4e05b11eba2f3f9d0a53a50ebf65d13ca5 config-baseline.plugin.json +e4f096d6d3c39fefdd1649cab8336b8dfba1f2194eb804fa6126eb25fe2d2b93 config-baseline.plugin.json diff --git a/docs/concepts/experimental-features.md b/docs/concepts/experimental-features.md index 9eb75780502..217978461fb 100644 --- a/docs/concepts/experimental-features.md +++ b/docs/concepts/experimental-features.md @@ -7,9 +7,9 @@ read_when: - You want one place to find the currently documented experimental flags --- -Experimental features are opt-in preview surfaces behind explicit flags. They need more real-world mileage before they get a stable default or a long-lived contract. +Experimental features are preview surfaces behind explicit flags. They need more real-world mileage before they get a stable default or a long-lived contract. -- Off by default unless a doc tells you to enable one. +- Off by default unless a doc describes a narrow automatic setup rule. - Shape and behavior can change faster than stable config. - Prefer a stable path when one already exists. - Roll out broadly only after testing in a smaller environment first. @@ -27,6 +27,8 @@ Experimental features are opt-in preview surfaces behind explicit flags. They ne `agents.defaults.experimental.localModelLean: true` drops heavyweight optional tools from the agent's direct surface every turn: `browser`, `cron`, `message`, `image_generate`, `music_generate`, `video_generate`, `tts`, and `pdf`. Explicitly allowed or delivery-required tools remain available, though Tool Search may catalog them instead of exposing them directly. Lean mode also defaults plugin/MCP/client catalogs to structured Tool Search (`tool_search`, `tool_describe`, `tool_call`) when `tools.toolSearch` is not already set. Use `agents.list[].experimental.localModelLean` to scope this to one agent. +During onboarding, a verified `ollama` or `lmstudio` inference route automatically sets `agents.defaults.experimental.localModelLean: true` when that value is absent. OpenClaw records that the setting came from onboarding, so a later verified non-local route lifts only the automatic setting. An explicitly configured `true` or `false` is preserved. Other self-hosted and OpenAI-compatible providers are not inferred from model names or URLs. + If you already tune Tool Search globally, OpenClaw leaves that config alone. Set `tools.toolSearch: false` to opt out of the lean-mode Tool Search default. In structured `tools` mode, lean runs keep `exec` directly visible beside the Tool Search controls so coding-tuned local models can still choose their familiar shell path. This changes schema visibility only: normal tool policy, sandboxing, and exec approvals still apply. Explicit `code` and `directory` modes keep their normal compaction behavior. diff --git a/src/commands/onboard-guided.custodian.test.ts b/src/commands/onboard-guided.custodian.test.ts index a03c85f9d12..003bd5b2942 100644 --- a/src/commands/onboard-guided.custodian.test.ts +++ b/src/commands/onboard-guided.custodian.test.ts @@ -236,6 +236,28 @@ describe("runGuidedOnboarding custodian flow", () => { expect(deps.launchHatchTui).toHaveBeenCalledWith("/tmp/work"); }); + it("announces the lean surface after local-model activation", async () => { + const announcement = + "This model is small, so I set up the lean surface — switching to a bigger model later lifts it."; + const prompter = createWizardPrompter(); + const deps = setupDeps({ + prompter, + activate: vi.fn(async () => ({ + ok: true as const, + modelRef: "lmstudio/qwen-local", + latencyMs: 1250, + lines: ["Inference verified: lmstudio/qwen-local", announcement], + })), + }); + + await runGuidedOnboarding({ acceptRisk: true, workspace: "/tmp/work" }, makeRuntime(), deps); + + expect(prompter.note).toHaveBeenCalledWith( + expect.stringContaining(announcement), + "Inference ready", + ); + }); + it("skips persisting an unchanged access mode", async () => { readConfigFileSnapshot.mockResolvedValue({ exists: true, diff --git a/src/config/local-model-lean-auto.test.ts b/src/config/local-model-lean-auto.test.ts new file mode 100644 index 00000000000..73b20c6fdff --- /dev/null +++ b/src/config/local-model-lean-auto.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import { applyAutoLocalModelLean } from "./local-model-lean-auto.js"; + +describe("local model lean onboarding defaults", () => { + it.each([ + ["ollama", true], + ["OLLAMA", true], + ["lmstudio", true], + ["ollama-cloud", false], + ["sglang", false], + ["vllm", false], + ["openai", false], + ])("classifies %s conservatively", (providerId, expected) => { + const modelRef = `${providerId}/test-model`; + const result = applyAutoLocalModelLean({ config: {}, providerId, modelRef }); + + expect(result.enabled).toBe(expected); + expect(result.changed).toBe(expected); + expect(result.config.agents?.defaults?.experimental?.localModelLean).toBe( + expected ? true : undefined, + ); + expect(result.config.wizard?.localModelLeanAutoModel).toBe(expected ? modelRef : undefined); + }); + + it.each([false, true])("preserves an explicit localModelLean=%s", (localModelLean) => { + const config = { agents: { defaults: { experimental: { localModelLean } } } }; + + expect( + applyAutoLocalModelLean({ config, providerId: "ollama", modelRef: "ollama/test-model" }), + ).toEqual({ + config, + changed: false, + enabled: false, + }); + }); + + it("lifts only an onboarding-owned lean setting for a later non-local route", () => { + const config = { + wizard: { localModelLeanAutoModel: "ollama/test-model" }, + agents: { + defaults: { + model: "ollama/test-model", + experimental: { localModelLean: true }, + }, + }, + }; + const lifted = applyAutoLocalModelLean({ + config, + providerId: "openai", + modelRef: "openai/gpt-test", + }); + + expect(lifted.changed).toBe(true); + expect(lifted.enabled).toBe(false); + expect(lifted.config.agents?.defaults?.experimental?.localModelLean).toBeUndefined(); + expect(lifted.config.wizard?.localModelLeanAutoModel).toBeUndefined(); + }); + + it("preserves an explicit lean setting for a non-local route", () => { + const config = { agents: { defaults: { experimental: { localModelLean: true } } } }; + + expect( + applyAutoLocalModelLean({ config, providerId: "openai", modelRef: "openai/gpt-test" }), + ).toEqual({ + config, + changed: false, + enabled: false, + }); + }); + + it("hands ownership to a model changed outside onboarding", () => { + const config = { + wizard: { localModelLeanAutoModel: "ollama/old-model" }, + agents: { + defaults: { + model: "openai/gpt-test", + experimental: { localModelLean: true }, + }, + }, + }; + + const result = applyAutoLocalModelLean({ + config, + providerId: "openai", + modelRef: "openai/gpt-test", + }); + + expect(result.config.agents?.defaults?.experimental?.localModelLean).toBe(true); + expect(result.config.wizard?.localModelLeanAutoModel).toBeUndefined(); + }); +}); diff --git a/src/config/local-model-lean-auto.ts b/src/config/local-model-lean-auto.ts new file mode 100644 index 00000000000..65f3c5fe3d9 --- /dev/null +++ b/src/config/local-model-lean-auto.ts @@ -0,0 +1,101 @@ +import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; +import type { OpenClawConfig } from "./types.openclaw.js"; + +const AUTO_LOCAL_MODEL_LEAN_PROVIDER_IDS = new Set(["lmstudio", "ollama"]); + +/** Returns true only for local runtimes that onboarding can identify without model-name guesses. */ +function shouldAutoEnableLocalModelLean(providerId: string): boolean { + return AUTO_LOCAL_MODEL_LEAN_PROVIDER_IDS.has(normalizeProviderId(providerId)); +} + +function resolveDefaultModelRef(config: OpenClawConfig): string | undefined { + const model = config.agents?.defaults?.model; + return typeof model === "string" ? model : model?.primary; +} + +function clearAutoModel(config: OpenClawConfig): OpenClawConfig { + const wizard = { ...config.wizard }; + delete wizard.localModelLeanAutoModel; + return { ...config, wizard }; +} + +/** Maintains the onboarding-owned lean default while preserving explicit user configuration. */ +export function applyAutoLocalModelLean(params: { + config: OpenClawConfig; + providerId: string; + modelRef: string; +}): { + config: OpenClawConfig; + changed: boolean; + enabled: boolean; +} { + const localModelLean = params.config.agents?.defaults?.experimental?.localModelLean; + const autoModel = params.config.wizard?.localModelLeanAutoModel; + const onboardingOwnsSetting = + autoModel !== undefined && resolveDefaultModelRef(params.config) === autoModel; + if (!shouldAutoEnableLocalModelLean(params.providerId)) { + if (!autoModel) { + return { config: params.config, changed: false, enabled: false }; + } + const config = clearAutoModel(params.config); + if (!onboardingOwnsSetting || localModelLean !== true) { + return { config, changed: true, enabled: false }; + } + const experimental = { ...params.config.agents?.defaults?.experimental }; + delete experimental.localModelLean; + return { + config: { + ...config, + agents: { + ...config.agents, + defaults: { + ...config.agents?.defaults, + experimental, + }, + }, + }, + changed: true, + enabled: false, + }; + } + if (localModelLean !== undefined) { + if (!autoModel) { + return { config: params.config, changed: false, enabled: false }; + } + if (!onboardingOwnsSetting || !localModelLean) { + return { config: clearAutoModel(params.config), changed: true, enabled: false }; + } + if (autoModel === params.modelRef) { + return { config: params.config, changed: false, enabled: false }; + } + return { + config: { + ...params.config, + wizard: { ...params.config.wizard, localModelLeanAutoModel: params.modelRef }, + }, + changed: true, + enabled: false, + }; + } + return { + config: { + ...params.config, + wizard: { + ...params.config.wizard, + localModelLeanAutoModel: params.modelRef, + }, + agents: { + ...params.config.agents, + defaults: { + ...params.config.agents?.defaults, + experimental: { + ...params.config.agents?.defaults?.experimental, + localModelLean: true, + }, + }, + }, + }, + changed: true, + enabled: true, + }; +} diff --git a/src/config/schema.help.core.ts b/src/config/schema.help.core.ts index 62ee40aa7f5..31c39cd4f1f 100644 --- a/src/config/schema.help.core.ts +++ b/src/config/schema.help.core.ts @@ -54,6 +54,8 @@ export const CORE_FIELD_HELP: Record = { "Command invocation recorded for the latest wizard run to preserve execution context. Use this to reproduce setup steps when verifying setup regressions.", "wizard.lastRunMode": 'Wizard execution mode recorded as "local" or "remote" for the most recent setup flow. Use this to understand whether setup targeted direct local runtime or remote gateway topology.', + "wizard.localModelLeanAutoModel": + "System-managed model reference indicating that inference onboarding enabled the global lean-model surface. A later verified route uses this marker to remove only the onboarding-owned setting; a model changed outside onboarding hands ownership back to the user.", "wizard.securityAcknowledgedAt": "ISO timestamp for when the setup security acknowledgement was accepted on this config. Setup uses this to avoid repeating the acknowledgement on later wizard runs.", audit: diff --git a/src/config/schema.labels.ts b/src/config/schema.labels.ts index 1f2b15646ea..3c54500c0e2 100644 --- a/src/config/schema.labels.ts +++ b/src/config/schema.labels.ts @@ -31,6 +31,7 @@ export const FIELD_LABELS: Record = { "wizard.lastRunCommit": "Wizard Last Run Commit", "wizard.lastRunCommand": "Wizard Last Run Command", "wizard.lastRunMode": "Wizard Last Run Mode", + "wizard.localModelLeanAutoModel": "Wizard Auto-Set Lean Model", "wizard.securityAcknowledgedAt": "Wizard Security Acknowledgement Timestamp", audit: "Audit Ledger", "audit.enabled": "Audit Ledger Enabled", diff --git a/src/config/types.openclaw.ts b/src/config/types.openclaw.ts index 8b358c259ad..dda7d445f9c 100644 --- a/src/config/types.openclaw.ts +++ b/src/config/types.openclaw.ts @@ -143,6 +143,8 @@ export type OpenClawConfig = { lastRunCommand?: string; /** Whether the last wizard run configured a local or remote install. */ lastRunMode?: "local" | "remote"; + /** Model whose lean-mode default is owned by inference onboarding. */ + localModelLeanAutoModel?: string; /** ISO timestamp when the setup security acknowledgement was accepted on this config. */ securityAcknowledgedAt?: string; }; diff --git a/src/config/zod-schema.root-shape.ts b/src/config/zod-schema.root-shape.ts index c34b7510b60..08dc934639b 100644 --- a/src/config/zod-schema.root-shape.ts +++ b/src/config/zod-schema.root-shape.ts @@ -82,6 +82,7 @@ export const OpenClawSchemaShape = { lastRunCommit: z.string().optional(), lastRunCommand: z.string().optional(), lastRunMode: z.union([z.literal("local"), z.literal("remote")]).optional(), + localModelLeanAutoModel: z.string().optional(), securityAcknowledgedAt: z.string().optional(), }) .optional(), diff --git a/src/system-agent/setup-inference.test.ts b/src/system-agent/setup-inference.test.ts index 7c16f60f74d..aa585c4efeb 100644 --- a/src/system-agent/setup-inference.test.ts +++ b/src/system-agent/setup-inference.test.ts @@ -1216,7 +1216,22 @@ describe("activateSetupInference", () => { expect(persistedConfig.gateway).toBeUndefined(); }); - it("rechecks the exact provider model and activates it without storing credentials", async () => { + it.each([ + { + name: "auto-enables the lean surface for a verified local model", + initialConfig: {} satisfies OpenClawConfig, + expectedLean: true, + expectedAnnouncement: true, + }, + { + name: "preserves an explicit localModelLean=false", + initialConfig: { + agents: { defaults: { experimental: { localModelLean: false } } }, + } satisfies OpenClawConfig, + expectedLean: false, + expectedAnnouncement: false, + }, + ])("$name", async ({ initialConfig, expectedLean, expectedAnnouncement }) => { const modelRef = "lmstudio/qwen-local"; const detect = vi.fn(async () => ({ modelRef, detail: "qwen-local at localhost" })); const prepare = vi.fn(async () => ({ @@ -1259,7 +1274,7 @@ describe("activateSetupInference", () => { }, ], }; - const configHarness = createConfigTransformHarness(); + const configHarness = createConfigTransformHarness(initialConfig); const updateAuthStore = vi.fn(); const result = await activateSetupInference({ @@ -1268,6 +1283,15 @@ describe("activateSetupInference", () => { surface: "gateway", runtime, deps: { + readConfigFileSnapshot: vi.fn(async () => ({ + exists: true, + valid: true, + path: "/tmp/openclaw.json", + issues: [], + config: initialConfig, + sourceConfig: initialConfig, + runtimeConfig: initialConfig, + })) as never, resolveManifestProviderAuthChoice: () => ({ pluginId: "lmstudio", providerId: "lmstudio", @@ -1285,11 +1309,23 @@ describe("activateSetupInference", () => { }); expect(result).toMatchObject({ ok: true, modelRef }); + if (!result.ok) { + throw new Error(result.error); + } + expect(result.lines).toEqual([ + `Inference verified: ${modelRef}`, + ...(expectedAnnouncement + ? [ + "This model is small, so I set up the lean surface — switching to a bigger model later lifts it.", + ] + : []), + ]); expect(detect).not.toHaveBeenCalled(); expect(prepare).toHaveBeenCalledOnce(); expect(updateAuthStore).not.toHaveBeenCalled(); expect(configHarness.current()).toMatchObject({ - agents: { defaults: { model: modelRef } }, + ...(expectedAnnouncement ? { wizard: { localModelLeanAutoModel: modelRef } } : {}), + agents: { defaults: { model: modelRef, experimental: { localModelLean: expectedLean } } }, models: { providers: { lmstudio: { @@ -1482,8 +1518,15 @@ describe("activateSetupInference", () => { expect(configHarness.current()).toEqual(concurrentConfig); }); - it("preserves authored provider rows instead of runtime-materialized metadata", async () => { + it("preserves authored provider rows and lifts an onboarding-owned lean setting", async () => { const sourceConfig = { + wizard: { localModelLeanAutoModel: "lmstudio/qwen-local" }, + agents: { + defaults: { + model: "lmstudio/qwen-local", + experimental: { localModelLean: true }, + }, + }, models: { providers: { openai: { @@ -1539,6 +1582,8 @@ describe("activateSetupInference", () => { expect(configHarness.current().models?.providers?.openai?.models).toEqual( sourceConfig.models.providers.openai.models, ); + expect(configHarness.current().agents?.defaults?.experimental?.localModelLean).toBeUndefined(); + expect(configHarness.current().wizard?.localModelLeanAutoModel).toBeUndefined(); }); it("rejects an existing route that changes after its live probe", async () => { diff --git a/src/system-agent/setup-inference.ts b/src/system-agent/setup-inference.ts index 8aa873067f6..cb17aedb728 100644 --- a/src/system-agent/setup-inference.ts +++ b/src/system-agent/setup-inference.ts @@ -35,6 +35,7 @@ import { type InferenceBackendKind, } from "../commands/onboard-inference.js"; import { createMergePatch } from "../config/io.write-prepare.js"; +import { applyAutoLocalModelLean } from "../config/local-model-lean-auto.js"; import { applyMergePatch } from "../config/merge-patch.js"; import { normalizeAgentModelRefForConfig, @@ -112,6 +113,8 @@ const log = createSubsystemLogger("system-agent/setup-inference"); export const SETUP_INFERENCE_TEST_TIMEOUT_MS = 90_000; const SETUP_INFERENCE_TEST_PROMPT = "Reply with the single word OK. Do not use tools."; const PROVIDER_AUTO_SETUP_KIND_PREFIX = "provider-auto:"; +const AUTO_LOCAL_MODEL_LEAN_ANNOUNCEMENT = + "This model is small, so I set up the lean surface — switching to a bigger model later lifts it."; export type ProviderAutoSetupInferenceKind = `provider-auto:${string}`; export type SetupInferenceKind = InferenceBackendKind | ProviderAutoSetupInferenceKind; @@ -1907,11 +1910,17 @@ async function activateSetupInferenceUnredacted( }; } + const autoLocalModelLeanUpdate = applyAutoLocalModelLean({ + config: sourceCfg, + providerId: testPlan.provider, + modelRef: plan.modelRef, + }); const needsPersistence = plan.persistModelRef !== undefined || plan.manualAuth !== undefined || codexPluginPatch !== undefined || - pendingCodexInstall !== undefined; + pendingCodexInstall !== undefined || + autoLocalModelLeanUpdate.changed; if ( !test.auth.authFingerprint && (!test.auth.runtimeOwnerFingerprint || @@ -1966,6 +1975,7 @@ async function activateSetupInferenceUnredacted( } } let committedConfig: OpenClawConfig | undefined; + let autoLocalModelLeanApplied = false; if (!needsPersistence) { const latestSnapshot = await readSnapshot(); const latestRuntime = @@ -2050,6 +2060,11 @@ async function activateSetupInferenceUnredacted( } next = enabledCodex.config; } + next = applyAutoLocalModelLean({ + config: next, + providerId: testPlan.provider, + modelRef: plan.modelRef, + }).config; next = selectModel ? selectModel(next) : next; if (!pendingCodexInstall) { return next; @@ -2185,6 +2200,11 @@ async function activateSetupInferenceUnredacted( "The authored target model metadata changed during its live inference test, so the verified candidate was not saved. Review the current model settings and retry.", ); } + const autoLocalModelLean = applyAutoLocalModelLean({ + config: current, + providerId: testPlan.provider, + modelRef: plan.modelRef, + }); const nextConfig = stageCandidate(current, "source"); const nextRouteProjection = await projectDefaultInferenceRoute(nextConfig); const nextResolvedRoute = await resolveSystemAgentConfiguredRouteFromConfig(nextConfig); @@ -2211,6 +2231,7 @@ async function activateSetupInferenceUnredacted( throwIfSetupInferenceCancelled(params); params.onCommitStarted?.(); commitMayHaveStarted = true; + autoLocalModelLeanApplied = autoLocalModelLean.enabled; return { nextConfig }; }, }); @@ -2284,7 +2305,13 @@ async function activateSetupInferenceUnredacted( ); } } - let lines = [`Inference verified: ${plan.modelRef}`]; + const announceAutoLocalModelLean = + autoLocalModelLeanApplied && + committedConfig?.agents?.defaults?.experimental?.localModelLean === true; + let lines = [ + `Inference verified: ${plan.modelRef}`, + ...(announceAutoLocalModelLean ? [AUTO_LOCAL_MODEL_LEAN_ANNOUNCEMENT] : []), + ]; if (params.surface === "gateway" && params.recordSetupAudit !== false) { const after = await readSnapshot().catch(() => null); try { From 35754a63ccafc41122085bc9467fd3e1dbc8d4f3 Mon Sep 17 00:00:00 2001 From: cxbAsDev Date: Sat, 18 Jul 2026 17:34:00 +0800 Subject: [PATCH 02/76] fix(plugins): bound marketplace manifest file reads (#101774) * fix(plugins): bound marketplace manifest file reads * fix(plugins): distinguish oversized marketplace manifest from other read failures * fix(plugins): follow symlinked marketplace manifests while bounding reads * test(plugins): cover oversized symlinked marketplace manifest target --- src/plugins/marketplace.test.ts | 61 +++++++++++++++++++++++++++++++++ src/plugins/marketplace.ts | 25 +++++++++++++- 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/plugins/marketplace.test.ts b/src/plugins/marketplace.test.ts index b7db20bd94a..6f7d6badac6 100644 --- a/src/plugins/marketplace.test.ts +++ b/src/plugins/marketplace.test.ts @@ -305,6 +305,67 @@ describe("marketplace plugins", () => { }); }); + it("rejects oversized local marketplace manifests", async () => { + await withTempDir("openclaw-marketplace-test-", async (rootDir) => { + const manifestPath = path.join(rootDir, ".claude-plugin", "marketplace.json"); + await fs.mkdir(path.dirname(manifestPath), { recursive: true }); + await fs.writeFile(manifestPath, Buffer.alloc(16 * 1024 * 1024 + 1, "x")); + + const result = await listMarketplacePlugins({ marketplace: rootDir }); + + expect(result).toEqual({ + ok: false, + error: "Marketplace manifest too large", + }); + }); + }); + + it("follows a symlinked marketplace manifest to a regular file", async () => { + if (process.platform === "win32") { + // Symlink support in unit tests is not guaranteed on Windows CI runners. + return; + } + await withTempDir("openclaw-marketplace-test-", async (rootDir) => { + const manifestPath = path.join(rootDir, ".claude-plugin", "marketplace.json"); + await fs.mkdir(path.dirname(manifestPath), { recursive: true }); + const targetPath = path.join(rootDir, "real-manifest.json"); + await fs.writeFile( + targetPath, + JSON.stringify({ plugins: [{ name: "symlinked-plugin", source: "." }] }), + "utf-8", + ); + await fs.symlink(targetPath, manifestPath); + + const result = await listMarketplacePlugins({ marketplace: rootDir }); + + expect(result.ok).toBe(true); + expect( + (result as { ok: true; manifest: { plugins: unknown[] } }).manifest.plugins, + ).toHaveLength(1); + }); + }); + + it("rejects a symlinked marketplace manifest whose target exceeds the size limit", async () => { + if (process.platform === "win32") { + // Symlink support in unit tests is not guaranteed on Windows CI runners. + return; + } + await withTempDir("openclaw-marketplace-test-", async (rootDir) => { + const manifestPath = path.join(rootDir, ".claude-plugin", "marketplace.json"); + await fs.mkdir(path.dirname(manifestPath), { recursive: true }); + const targetPath = path.join(rootDir, "real-manifest.json"); + await fs.writeFile(targetPath, Buffer.alloc(16 * 1024 * 1024 + 1, "x")); + await fs.symlink(targetPath, manifestPath); + + const result = await listMarketplacePlugins({ marketplace: rootDir }); + + expect(result).toEqual({ + ok: false, + error: "Marketplace manifest too large", + }); + }); + }); + it("resolves relative plugin paths against the marketplace root", async () => { await withTempDir("openclaw-marketplace-test-", async (rootDir) => { const pluginDir = path.join(rootDir, "plugins", "frontend-design"); diff --git a/src/plugins/marketplace.ts b/src/plugins/marketplace.ts index 6a425dd14cf..c670aa2882b 100644 --- a/src/plugins/marketplace.ts +++ b/src/plugins/marketplace.ts @@ -13,6 +13,7 @@ import { resolveOsHomeRelativePath } from "../infra/home-dir.js"; import { tryReadJson } from "../infra/json-files.js"; import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js"; import { isPathInside } from "../infra/path-guards.js"; +import { readRegularFile } from "../infra/regular-file.js"; import { runCommandWithTimeout } from "../process/exec.js"; import type { InstallPolicySource } from "../security/install-policy.js"; import { resolveUserPath } from "../utils.js"; @@ -23,6 +24,7 @@ import { installPluginFromPath, type InstallPluginResult } from "./install.js"; const DEFAULT_GIT_TIMEOUT_MS = 120_000; const DEFAULT_MARKETPLACE_DOWNLOAD_TIMEOUT_MS = 120_000; const MAX_MARKETPLACE_ARCHIVE_BYTES = 256 * 1024 * 1024; +const MAX_MARKETPLACE_MANIFEST_BYTES = 16 * 1024 * 1024; const MARKETPLACE_MANIFEST_CANDIDATES = [ path.join(".claude-plugin", "marketplace.json"), "marketplace.json", @@ -597,7 +599,28 @@ async function loadMarketplace(params: { remoteRef?: string; cleanup?: () => Promise; }): Promise<{ ok: true; marketplace: LoadedMarketplace } | { ok: false; error: string }> => { - const raw = await fs.readFile(paramsLocal.manifestPath, "utf-8"); + let raw: string; + try { + // Resolve symlinks so a marketplace.json that points to a regular file + // keeps working, while the bounded regular-file read still rejects + // directories, FIFOs, and oversized targets. + const resolvedManifestPath = await fs.realpath(paramsLocal.manifestPath); + const { buffer } = await readRegularFile({ + filePath: resolvedManifestPath, + maxBytes: MAX_MARKETPLACE_MANIFEST_BYTES, + }); + raw = buffer.toString("utf-8"); + } catch (err) { + await paramsLocal.cleanup?.(); + const message = err instanceof Error ? err.message : String(err); + // readRegularFile rejects symlinks/non-files and caps file size. Only the + // size cap should be reported as an oversize manifest; other read failures + // need their own diagnostic so users don't chase the wrong problem. + if (message.startsWith("File exceeds")) { + return { ok: false, error: "Marketplace manifest too large" }; + } + return { ok: false, error: `Marketplace manifest unreadable: ${message}` }; + } const parsed = parseMarketplaceManifest(raw, paramsLocal.manifestPath); if (!parsed.ok) { await paramsLocal.cleanup?.(); From 609e5d23184bb849fa2efd9c2d4c945ac8c27c81 Mon Sep 17 00:00:00 2001 From: xingzhou Date: Sat, 18 Jul 2026 17:36:35 +0800 Subject: [PATCH 03/76] fix(gateway): session workspace reveal hangs on foreground opener (#110277) * fix(gateway): bound session workspace reveal opener * refactor(gateway): share bounded path opener lifecycle --------- Co-authored-by: Peter Steinberger --- src/gateway/server-methods/config.test.ts | 54 ++------ src/gateway/server-methods/open-path.test.ts | 138 +++++++++++++++++++ src/gateway/server-methods/open-path.ts | 86 +++++++++++- 3 files changed, 235 insertions(+), 43 deletions(-) create mode 100644 src/gateway/server-methods/open-path.test.ts diff --git a/src/gateway/server-methods/config.test.ts b/src/gateway/server-methods/config.test.ts index 5cbf9d2bf4c..3a79387678e 100644 --- a/src/gateway/server-methods/config.test.ts +++ b/src/gateway/server-methods/config.test.ts @@ -11,10 +11,9 @@ import { loadConfigSchemaResponseForTests, } from "./config.js"; import { createConfigHandlerHarness } from "./config.test-helpers.js"; -import { resolveOpenPathCommand } from "./open-path.js"; -const { runExecMock, loadGatewayRuntimeConfigSchemaMock } = vi.hoisted(() => ({ - runExecMock: vi.fn(), +const { execOpenPathMock, loadGatewayRuntimeConfigSchemaMock } = vi.hoisted(() => ({ + execOpenPathMock: vi.fn(), loadGatewayRuntimeConfigSchemaMock: vi.fn(() => ({ schema: { type: "object" }, uiHints: undefined, @@ -22,14 +21,17 @@ const { runExecMock, loadGatewayRuntimeConfigSchemaMock } = vi.hoisted(() => ({ })), })); -vi.mock("../../process/exec.js", () => ({ runExec: runExecMock })); +vi.mock("./open-path.js", async () => { + const actual = await vi.importActual("./open-path.js"); + return { ...actual, execOpenPath: execOpenPathMock }; +}); vi.mock("../../config/runtime-schema.js", () => ({ loadGatewayRuntimeConfigSchema: loadGatewayRuntimeConfigSchemaMock, })); -function mockRunExecError(error: Error) { - runExecMock.mockRejectedValue(error); +function mockOpenPathError(error: Error) { + execOpenPathMock.mockRejectedValue(error); } async function invokeConfigOpenFile() { @@ -47,40 +49,12 @@ afterEach(() => { vi.clearAllMocks(); }); -describe("resolveOpenPathCommand", () => { - it("uses open on macOS", () => { - expect(resolveOpenPathCommand("/tmp/openclaw.json", "darwin")).toEqual({ - command: "open", - args: ["/tmp/openclaw.json"], - }); - }); - - it("uses xdg-open on Linux", () => { - expect(resolveOpenPathCommand("/tmp/openclaw.json", "linux")).toEqual({ - command: "xdg-open", - args: ["/tmp/openclaw.json"], - }); - }); - - it("uses a quoted PowerShell FilePath on Windows", () => { - expect(resolveOpenPathCommand(String.raw`C:\tmp\o'hai & calc.json`, "win32")).toEqual({ - command: "powershell.exe", - args: [ - "-NoProfile", - "-NonInteractive", - "-Command", - String.raw`Start-Process -FilePath 'C:\tmp\o''hai & calc.json'`, - ], - }); - }); -}); - describe("config.openFile", () => { it("opens the configured file without shell interpolation", async () => { await withEnvAsync({ OPENCLAW_CONFIG_PATH: "/tmp/config $(touch pwned).json" }, async () => { - runExecMock.mockImplementation(async (command: string, args: string[]) => { - expect(["open", "xdg-open", "powershell.exe"]).toContain(command); - expect(args).toEqual(["/tmp/config $(touch pwned).json"]); + execOpenPathMock.mockImplementation(async (command: { command: string; args: string[] }) => { + expect(["open", "xdg-open", "powershell.exe"]).toContain(command.command); + expect(command.args).toEqual(["/tmp/config $(touch pwned).json"]); return { stdout: "", stderr: "" }; }); @@ -99,7 +73,7 @@ describe("config.openFile", () => { it("returns a detailed error and logs details when the opener fails", async () => { await withEnvAsync({ OPENCLAW_CONFIG_PATH: "/tmp/config.json" }, async () => { - mockRunExecError(Object.assign(new Error("spawn xdg-open ENOENT"), { code: "ENOENT" })); + mockOpenPathError(Object.assign(new Error("spawn xdg-open ENOENT"), { code: "ENOENT" })); const { respond, logGateway } = await invokeConfigOpenFile(); @@ -121,7 +95,7 @@ describe("config.openFile", () => { it("does not split surrogate pairs when truncating the failed config path", async () => { const pathPrefix = `/tmp/${"a".repeat(111)}`; await withEnvAsync({ OPENCLAW_CONFIG_PATH: `${pathPrefix}😀tail.json` }, async () => { - mockRunExecError(new Error("open failed")); + mockOpenPathError(new Error("open failed")); const { logGateway } = await invokeConfigOpenFile(); @@ -133,7 +107,7 @@ describe("config.openFile", () => { it("returns actionable headless environment error when xdg-open reports no method available", async () => { await withEnvAsync({ OPENCLAW_CONFIG_PATH: "/tmp/config.json" }, async () => { - mockRunExecError(new Error("xdg-open: no method available for opening '/tmp/config.json'")); + mockOpenPathError(new Error("xdg-open: no method available for opening '/tmp/config.json'")); const { respond, logGateway } = await invokeConfigOpenFile(); diff --git a/src/gateway/server-methods/open-path.test.ts b/src/gateway/server-methods/open-path.test.ts new file mode 100644 index 00000000000..5a58ad6849f --- /dev/null +++ b/src/gateway/server-methods/open-path.test.ts @@ -0,0 +1,138 @@ +import { PassThrough } from "node:stream"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const { runExecMock, spawnCommandMock } = vi.hoisted(() => ({ + runExecMock: vi.fn(), + spawnCommandMock: vi.fn(), +})); + +vi.mock("../../process/exec.js", () => ({ + runExec: runExecMock, + spawnCommand: spawnCommandMock, +})); + +import { execOpenPath, resolveOpenPathCommand } from "./open-path.js"; + +function fakeChild(result: Promise) { + const unref = vi.fn(); + const kill = vi.fn(); + const stderr = new PassThrough(); + return { + child: Object.assign(result, { kill, stderr, unref }), + kill, + stderr, + unref, + }; +} + +afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); +}); + +describe("resolveOpenPathCommand", () => { + it("uses open on macOS", () => { + expect(resolveOpenPathCommand("/tmp/openclaw.json", "darwin")).toEqual({ + command: "open", + args: ["/tmp/openclaw.json"], + }); + }); + + it("uses xdg-open on Linux", () => { + expect(resolveOpenPathCommand("/tmp/openclaw.json", "linux")).toEqual({ + command: "xdg-open", + args: ["/tmp/openclaw.json"], + }); + }); + + it("uses a quoted PowerShell FilePath on Windows", () => { + expect(resolveOpenPathCommand(String.raw`C:\tmp\o'hai & calc.json`, "win32")).toEqual({ + command: "powershell.exe", + args: [ + "-NoProfile", + "-NonInteractive", + "-Command", + String.raw`Start-Process -FilePath 'C:\tmp\o''hai & calc.json'`, + ], + }); + }); +}); + +describe("execOpenPath", () => { + it.each(["darwin", "win32"] as const)("bounds the %s launcher wait", async (platform) => { + runExecMock.mockResolvedValue({ stdout: "", stderr: "" }); + const command = { + command: platform === "darwin" ? "open" : "powershell.exe", + args: ["/tmp/workspace"], + }; + + await execOpenPath(command, platform); + + expect(runExecMock).toHaveBeenCalledWith(command.command, command.args, { + logOutput: false, + timeoutMs: 5_000, + }); + }); + + it("detaches xdg-open and preserves an immediate successful exit", async () => { + const spawned = fakeChild(Promise.resolve({ failed: false })); + spawnCommandMock.mockReturnValue(spawned.child); + + await execOpenPath({ command: "xdg-open", args: ["/tmp/workspace"] }, "linux"); + + expect(spawnCommandMock).toHaveBeenCalledWith(["xdg-open", "/tmp/workspace"], { + buffer: false, + cleanup: false, + detached: true, + reject: true, + stdio: ["ignore", "ignore", "pipe"], + }); + expect(spawned.unref).toHaveBeenCalledOnce(); + }); + + it("returns after startup observation without killing a foreground Linux handler", async () => { + vi.useFakeTimers(); + let settleChild: (value: unknown) => void = () => {}; + const childResult = new Promise((resolve) => { + settleChild = resolve; + }); + const spawned = fakeChild(childResult); + spawnCommandMock.mockReturnValue(spawned.child); + let settled = false; + + const execution = execOpenPath({ command: "xdg-open", args: ["/tmp/workspace"] }, "linux").then( + () => { + settled = true; + }, + ); + + await vi.advanceTimersByTimeAsync(4_999); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + await execution; + + expect(settled).toBe(true); + expect(spawned.kill).not.toHaveBeenCalled(); + expect(spawned.stderr.destroyed).toBe(false); + expect(spawned.stderr.write("foreground handler: delayed diagnostic")).toBe(true); + settleChild({ failed: false }); + await Promise.resolve(); + expect(spawned.stderr.destroyed).toBe(true); + }); + + it("propagates an immediate Linux launcher failure with bounded stderr", async () => { + let rejectChild: (error: Error) => void = () => {}; + const spawned = fakeChild( + new Promise((_, reject) => { + rejectChild = reject; + }), + ); + spawnCommandMock.mockReturnValue(spawned.child); + + const execution = execOpenPath({ command: "xdg-open", args: ["/tmp/workspace"] }, "linux"); + spawned.stderr.write("xdg-open: no method available for opening '/tmp/workspace'"); + rejectChild(new Error("Command failed with exit code 3: xdg-open")); + + await expect(execution).rejects.toThrow("xdg-open: no method available"); + }); +}); diff --git a/src/gateway/server-methods/open-path.ts b/src/gateway/server-methods/open-path.ts index a2d0926cc43..480240dc626 100644 --- a/src/gateway/server-methods/open-path.ts +++ b/src/gateway/server-methods/open-path.ts @@ -1,5 +1,9 @@ import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; -import { runExec } from "../../process/exec.js"; +import { runExec, spawnCommand } from "../../process/exec.js"; + +const OPEN_PATH_TIMEOUT_MS = 5_000; +const XDG_OPEN_STARTUP_OBSERVATION_MS = 5_000; +const XDG_OPEN_STDERR_MAX_CHARS = 4_096; type OpenPathCommand = { command: string; @@ -32,8 +36,84 @@ export function resolveOpenPathCommand( }; } -export async function execOpenPath(command: OpenPathCommand): Promise { - await runExec(command.command, command.args, { logOutput: false }); +async function observeXdgOpenStartup(command: OpenPathCommand): Promise { + // xdg-open can synchronously own a foreground application. Observe startup + // failures without making the Gateway own the launched application's lifetime. + const child = spawnCommand([command.command, ...command.args], { + buffer: false, + cleanup: false, + detached: true, + reject: true, + stdio: ["ignore", "ignore", "pipe"], + }); + child.unref(); + let stderrText = ""; + const stderr = child.stderr; + stderr?.setEncoding("utf8"); + const onStderr = (chunk: string | Buffer) => { + if (stderrText.length >= XDG_OPEN_STDERR_MAX_CHARS) { + return; + } + stderrText += String(chunk).slice(0, XDG_OPEN_STDERR_MAX_CHARS - stderrText.length); + }; + stderr?.on("data", onStderr); + + await new Promise((resolve, reject) => { + let observationComplete = false; + const releaseStderr = (childSettled: boolean) => { + stderr?.off("data", onStderr); + if (childSettled) { + stderr?.destroy(); + return; + } + // Keep draining after the observation window. Closing the pipe can send + // SIGPIPE to a foreground application that writes diagnostics later. + stderr?.resume(); + (stderr as (typeof stderr & { unref?: () => void }) | null)?.unref?.(); + }; + const timer = setTimeout(() => { + observationComplete = true; + releaseStderr(false); + resolve(); + }, XDG_OPEN_STARTUP_OBSERVATION_MS); + void child.then( + () => { + clearTimeout(timer); + releaseStderr(true); + if (!observationComplete) { + resolve(); + } + }, + (error: unknown) => { + clearTimeout(timer); + releaseStderr(true); + if (observationComplete) { + return; + } + const commandError = error instanceof Error ? error : new Error(String(error)); + const diagnostic = stderrText.trim(); + reject( + diagnostic + ? new Error(`${commandError.message}: ${diagnostic}`, { cause: commandError }) + : commandError, + ); + }, + ); + }); +} + +export async function execOpenPath( + command: OpenPathCommand, + platform: NodeJS.Platform = process.platform, +): Promise { + if (platform === "linux") { + await observeXdgOpenStartup(command); + return; + } + await runExec(command.command, command.args, { + logOutput: false, + timeoutMs: OPEN_PATH_TIMEOUT_MS, + }); } export function formatOpenPathError(error: unknown): string { From f14947d256d8238ed2fce3ca3ec81e4dae3fe8c2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 10:37:25 +0100 Subject: [PATCH 04/76] fix(codex): persist automatic compaction history (#110587) * fix(codex): persist automatic compaction counts * test(codex): stress compaction and restart recovery * test(codex): bind compaction waves to fresh counts * chore: keep Codex release note in PR --- docs/help/testing-live.md | 22 + .../src/app-server/event-projector.test.ts | 5 +- .../codex/src/app-server/event-projector.ts | 6 +- .../codex/src/app-server/run-attempt-start.ts | 7 +- scripts/test-live-codex-harness-docker.sh | 12 + ...gateway-codex-harness.live-helpers.test.ts | 25 + .../gateway-codex-harness.live-helpers.ts | 24 + .../gateway-codex-harness.live.test.ts | 477 +++++++++++++++--- .../test-live-codex-harness-docker.test.ts | 18 + 9 files changed, 510 insertions(+), 86 deletions(-) diff --git a/docs/help/testing-live.md b/docs/help/testing-live.md index c4ab7b43791..af947358fdb 100644 --- a/docs/help/testing-live.md +++ b/docs/help/testing-live.md @@ -334,6 +334,17 @@ Docker notes: history. Override the bounded counts with `OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS` (1-20) and `OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS` (1-10). +- Optional fan-out stress: set `OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE=1` + and `OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT` (1-12). The harness starts + every child concurrently, waits for every terminal run, and verifies each + unique child reply and native thread identity. +- Optional compaction stress: `OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS=1` + generates bounded native tool output, requires automatic compaction events, + verifies the persisted compaction count and hidden-marker recall, restarts + the Gateway and physical Codex app-server, then repeats the output and + compaction wave. Tune the bounded work with + `OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS` (1-8) and + `OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES` (100000-1000000). - Optional loop-relay opt-out probe: `OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY=1` - The requested thinking preference may map to the nearest effort advertised @@ -367,6 +378,17 @@ Restart and history stress: ```bash OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS=1 \ +pnpm test:docker:live-codex-harness +``` + +Fan-out, large-output, compaction, and restart stress: + +```bash +OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key \ + OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE=1 \ + OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT=8 \ + OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS=1 \ + OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS=1 \ pnpm test:docker:live-codex-harness ``` diff --git a/extensions/codex/src/app-server/event-projector.test.ts b/extensions/codex/src/app-server/event-projector.test.ts index 3e8ae5f6166..03856fcbcc2 100644 --- a/extensions/codex/src/app-server/event-projector.test.ts +++ b/extensions/codex/src/app-server/event-projector.test.ts @@ -2998,7 +2998,10 @@ describe("CodexAppServerEventProjector", () => { expect(JSON.stringify(result.messagesSnapshot[2])).toContain("Codex plan"); expect(JSON.stringify(result.messagesSnapshot[2])).toContain("next"); expect(JSON.stringify(result.messagesSnapshot[2])).toContain("[in_progress] patch"); - expect(requireRecord(result.itemLifecycle, "item lifecycle").compactionCount).toBe(1); + expect(result.compactionCount).toBe(1); + expect(requireRecord(result.itemLifecycle, "item lifecycle")).not.toHaveProperty( + "compactionCount", + ); expect(onContextCompacted).toHaveBeenCalledOnce(); }); diff --git a/extensions/codex/src/app-server/event-projector.ts b/extensions/codex/src/app-server/event-projector.ts index d894f3d89d7..8feada7e4b4 100644 --- a/extensions/codex/src/app-server/event-projector.ts +++ b/extensions/codex/src/app-server/event-projector.ts @@ -434,6 +434,9 @@ export class CodexAppServerEventProjector { successfulCronAdds: toolTelemetry.successfulCronAdds, cloudCodeAssistFormatError: false, attemptUsage: projectedUsage, + ...(this.completedCompactionCount > 0 + ? { compactionCount: this.completedCompactionCount } + : {}), replayMetadata: { hadPotentialSideEffects, replaySafe: !hadPotentialSideEffects, @@ -442,9 +445,6 @@ export class CodexAppServerEventProjector { startedCount: this.activeItemIds.size + this.completedItemIds.size, completedCount: this.completedItemIds.size, activeCount: this.activeItemIds.size, - ...(this.completedCompactionCount > 0 - ? { compactionCount: this.completedCompactionCount } - : {}), }, yieldDetected: options?.yieldDetected || false, didSendDeterministicApprovalPrompt: diff --git a/extensions/codex/src/app-server/run-attempt-start.ts b/extensions/codex/src/app-server/run-attempt-start.ts index 872c310ab3b..be79f0d33b4 100644 --- a/extensions/codex/src/app-server/run-attempt-start.ts +++ b/extensions/codex/src/app-server/run-attempt-start.ts @@ -159,7 +159,12 @@ export async function startCodexAttemptRuntime(resources: CodexAttemptResources) state.codexSandboxPolicy = startupResult.sandboxPolicy; void emitCodexAppServerEvent(params, { stream: "codex_app_server.lifecycle", - data: { phase: "thread_ready", threadId: state.thread.threadId }, + data: { + phase: "thread_ready", + threadId: state.thread.threadId, + action: state.thread.lifecycle.action, + clientId: state.client.getInstanceId(), + }, }); } catch (error) { activateNativePreToolUseFailureFallback(); diff --git a/scripts/test-live-codex-harness-docker.sh b/scripts/test-live-codex-harness-docker.sh index 0f06a4ea7ca..b8f5cfe4a8c 100644 --- a/scripts/test-live-codex-harness-docker.sh +++ b/scripts/test-live-codex-harness-docker.sh @@ -367,11 +367,17 @@ echo "==> Chat image probe: ${OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE:-0}" echo "==> Image probe: ${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}" echo "==> MCP probe: ${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}" echo "==> Subagent probe: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}" +echo "==> Subagent count: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}" echo "==> Subagent-only fast path: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-auto}" echo "==> Guardian probe: ${OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE:-1}" +echo "==> Code-mode-only probe: ${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}" +echo "==> Loop relay disabled: ${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}" echo "==> Resume stress: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS:-0}" echo "==> Resume stress history turns: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS:-4}" echo "==> Resume stress restarts: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}" +echo "==> Compaction stress: ${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}" +echo "==> Compaction stress turns: ${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}" +echo "==> Large output bytes: ${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}" echo "==> Auth mode: $CODEX_HARNESS_AUTH_MODE" echo "==> Profile file: $PROFILE_STATUS" echo "==> CI-safe Codex config: ${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}" @@ -401,9 +407,14 @@ DOCKER_RUN_ARGS+=(--rm -t \ -e OPENCLAW_LIVE_CODEX_HARNESS_AUTH="$CODEX_HARNESS_AUTH_MODE" \ -e OPENCLAW_LIVE_CODEX_HARNESS=1 \ -e OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_DEBUG="${OPENCLAW_LIVE_CODEX_HARNESS_DEBUG:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY="${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE:-1}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES="${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_MODEL="${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-openai/gpt-5.6-luna}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_TARGETS="${OPENCLAW_LIVE_CODEX_HARNESS_TARGETS:-}" \ @@ -416,6 +427,7 @@ DOCKER_RUN_ARGS+=(--rm -t \ -e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_SETUP_TIMEOUT_SECONDS="$CODEX_HARNESS_SETUP_TIMEOUT_SECONDS" \ -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG="${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}" \ -e OPENCLAW_LIVE_CODEX_CLI_PACKAGE_SPEC="$CODEX_CLI_PACKAGE_SPEC" \ diff --git a/src/gateway/gateway-codex-harness.live-helpers.test.ts b/src/gateway/gateway-codex-harness.live-helpers.test.ts index b2aa0136758..f173074350e 100644 --- a/src/gateway/gateway-codex-harness.live-helpers.test.ts +++ b/src/gateway/gateway-codex-harness.live-helpers.test.ts @@ -10,6 +10,7 @@ import { isExpectedYieldedAgentTimeout, isRetryableCodexHarnessLiveError, isStrictExpectedCodexModelsCommandText, + shouldUseCodexHarnessSubagentOnlyFastPath, } from "./gateway-codex-harness.live-helpers.js"; const includesExpectedCodexModelsCommandText = (text: string) => @@ -30,6 +31,30 @@ function expectStrictCodexModelsCommandText(text: string): void { } describe("gateway codex harness live helpers", () => { + it("keeps combined stress probes out of the subagent-only fast path", () => { + const base = { + chatImageProbe: false, + codeModeOnly: false, + compactionStress: false, + explicitOptOut: false, + guardianProbe: false, + imageProbe: false, + mcpProbe: false, + resumeStress: false, + subagentProbe: true, + }; + + expect(shouldUseCodexHarnessSubagentOnlyFastPath(base)).toBe(true); + expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, resumeStress: true })).toBe(false); + expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, compactionStress: true })).toBe( + false, + ); + expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, codeModeOnly: true })).toBe(false); + expect(shouldUseCodexHarnessSubagentOnlyFastPath({ ...base, explicitOptOut: true })).toBe( + false, + ); + }); + it("classifies sessions.list timeouts as retryable live Codex errors", () => { const error = new Error("gateway request timeout for sessions.list"); diff --git a/src/gateway/gateway-codex-harness.live-helpers.ts b/src/gateway/gateway-codex-harness.live-helpers.ts index 9b8c83b4014..2fec444a104 100644 --- a/src/gateway/gateway-codex-harness.live-helpers.ts +++ b/src/gateway/gateway-codex-harness.live-helpers.ts @@ -81,6 +81,30 @@ export const EXPECTED_CODEX_MODELS_COMMAND_TEXT = [ "Current OpenClaw session status reports the active model as:", ] as const; +export function shouldUseCodexHarnessSubagentOnlyFastPath(params: { + chatImageProbe: boolean; + codeModeOnly: boolean; + compactionStress: boolean; + explicitOptOut: boolean; + guardianProbe: boolean; + imageProbe: boolean; + mcpProbe: boolean; + resumeStress: boolean; + subagentProbe: boolean; +}): boolean { + return ( + params.subagentProbe && + !params.chatImageProbe && + !params.codeModeOnly && + !params.compactionStress && + !params.guardianProbe && + !params.imageProbe && + !params.mcpProbe && + !params.resumeStress && + !params.explicitOptOut + ); +} + const HEALTHY_CODEX_MODELS_COMMAND_TEXT = [ "Codex models:", "Available Codex models", diff --git a/src/gateway/gateway-codex-harness.live.test.ts b/src/gateway/gateway-codex-harness.live.test.ts index a781904dbe1..cf948c623cc 100644 --- a/src/gateway/gateway-codex-harness.live.test.ts +++ b/src/gateway/gateway-codex-harness.live.test.ts @@ -30,6 +30,7 @@ import { isExpectedYieldedAgentTimeout, isRetryableCodexHarnessLiveError, isStrictExpectedCodexModelsCommandText, + shouldUseCodexHarnessSubagentOnlyFastPath, } from "./gateway-codex-harness.live-helpers.js"; import { assertCronJobMatches, @@ -63,13 +64,6 @@ const CODEX_HARNESS_CODE_MODE_ONLY = isTruthyEnvValue( const CODEX_HARNESS_DISABLE_LOOP_RELAY = isTruthyEnvValue( process.env.OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY, ); -const CODEX_HARNESS_SUBAGENT_ONLY = - CODEX_HARNESS_SUBAGENT_PROBE && - !CODEX_HARNESS_CHAT_IMAGE_PROBE && - !CODEX_HARNESS_IMAGE_PROBE && - !CODEX_HARNESS_MCP_PROBE && - !CODEX_HARNESS_GUARDIAN_PROBE && - process.env.OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY !== "0"; const CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS = isTruthyEnvValue( process.env.OPENCLAW_LIVE_CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS, ); @@ -88,6 +82,40 @@ const CODEX_HARNESS_RESUME_STRESS_RESTARTS = resolveBoundedPositiveIntEnv( 3, 10, ); +const CODEX_HARNESS_COMPACTION_STRESS = isTruthyEnvValue( + process.env.OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS, +); +const CODEX_HARNESS_COMPACTION_STRESS_TURNS = resolveBoundedPositiveIntEnv( + "OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS", + process.env.OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS, + 4, + 8, +); +const CODEX_HARNESS_LARGE_OUTPUT_BYTES = resolveBoundedPositiveIntEnv( + "OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES", + process.env.OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES, + 300_000, + 1_000_000, + 100_000, +); +const CODEX_HARNESS_SUBAGENT_COUNT = resolveBoundedPositiveIntEnv( + "OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT", + process.env.OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT, + 1, + 12, +); +const CODEX_HARNESS_SUBAGENT_ONLY = shouldUseCodexHarnessSubagentOnlyFastPath({ + chatImageProbe: CODEX_HARNESS_CHAT_IMAGE_PROBE, + codeModeOnly: CODEX_HARNESS_CODE_MODE_ONLY, + compactionStress: CODEX_HARNESS_COMPACTION_STRESS, + explicitOptOut: process.env.OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY === "0", + guardianProbe: CODEX_HARNESS_GUARDIAN_PROBE, + imageProbe: CODEX_HARNESS_IMAGE_PROBE, + mcpProbe: CODEX_HARNESS_MCP_PROBE, + resumeStress: CODEX_HARNESS_RESUME_STRESS, + subagentProbe: CODEX_HARNESS_SUBAGENT_PROBE, +}); +const CODEX_HARNESS_RESTART_STRESS = CODEX_HARNESS_RESUME_STRESS || CODEX_HARNESS_COMPACTION_STRESS; const CODEX_HARNESS_REQUEST_TIMEOUT_MS = resolveLiveTimeoutMs( process.env.OPENCLAW_LIVE_CODEX_HARNESS_REQUEST_TIMEOUT_MS, 300_000, @@ -103,7 +131,7 @@ const CODEX_HARNESS_THINKING = resolveCodexHarnessThinkingLevel( ); const describeLive = LIVE && CODEX_HARNESS_LIVE ? describe : describe.skip; const describeDisabled = LIVE && !CODEX_HARNESS_LIVE ? describe : describe.skip; -const CODEX_HARNESS_TIMEOUT_MS = CODEX_HARNESS_RESUME_STRESS ? 1_800_000 : 900_000; +const CODEX_HARNESS_TIMEOUT_MS = CODEX_HARNESS_RESTART_STRESS ? 3_600_000 : 900_000; const DEFAULT_CODEX_MODEL = "openai/gpt-5.6-luna"; const GATEWAY_CONNECT_TIMEOUT_MS = 60_000; const CODEX_HARNESS_REASONING_EFFORTS = [ @@ -132,6 +160,8 @@ type CapturedAgentEvent = { }; const observedCodexThreadIds = new Map(); +const observedCodexClientIds = new Map(); +const observedCodexThreadActions = new Map(); type GuardianPluginApprovalDecision = "allow-once" | "deny"; type CodexHarnessThinkingLevel = @@ -154,6 +184,7 @@ function resolveBoundedPositiveIntEnv( raw: string | undefined, fallback: number, max: number, + min = 1, ): number { if (raw === undefined || raw.trim() === "") { return fallback; @@ -162,8 +193,8 @@ function resolveBoundedPositiveIntEnv( throw new Error(`invalid ${name}: ${raw}`); } const parsed = Number(raw); - if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > max) { - throw new Error(`${name} must be between 1 and ${max}`); + if (!Number.isSafeInteger(parsed) || parsed < min || parsed > max) { + throw new Error(`${name} must be between ${min} and ${max}`); } return parsed; } @@ -392,6 +423,7 @@ async function assertCodexHarnessTranscriptModelIdentity(params: { async function writeLiveGatewayConfig(params: { codexAppServerMode?: "guardian" | "yolo"; codeModeOnly?: boolean; + compactionStress?: boolean; loopDetectionPreToolUseRelay?: boolean; configPath: string; modelKey: string; @@ -414,6 +446,23 @@ async function writeLiveGatewayConfig(params: { config: { appServer: { mode: params.codexAppServerMode ?? "yolo", + ...(params.compactionStress + ? { + args: [ + "-c", + "model_auto_compact_token_limit_scope=body_after_prefix", + "-c", + // One truncated 300 KB tool result is only a few thousand + // tokens after Codex applies its transcript output cap. + "model_auto_compact_token_limit=4000", + "-c", + "tool_output_token_limit=10000", + "app-server", + "--listen", + "stdio://", + ], + } + : {}), ...(params.codeModeOnly === true ? { codeModeOnly: true } : {}), ...(params.loopDetectionPreToolUseRelay === false ? { loopDetectionPreToolUseRelay: false } @@ -430,6 +479,11 @@ async function writeLiveGatewayConfig(params: { workspace: params.workspace, skipBootstrap: true, timeoutSeconds: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS, + maxConcurrent: Math.max(4, CODEX_HARNESS_SUBAGENT_COUNT + 1), + subagents: { + maxConcurrent: CODEX_HARNESS_SUBAGENT_COUNT, + maxChildrenPerAgent: CODEX_HARNESS_SUBAGENT_COUNT, + }, thinkingDefault: CODEX_HARNESS_THINKING, model: { primary: params.modelKey }, models: { [params.modelKey]: { agentRuntime: { id: "codex" } } }, @@ -454,17 +508,18 @@ async function requestAgentTextWithEvents(params: { acceptYieldedTimeout?: boolean; client: GatewayClient; eventPrefix?: string; + eventPrefixes?: readonly string[]; includeAllSessions?: boolean; message: string; sessionKey: string; -}): Promise<{ text: string; events: CapturedAgentEvent[] }> { +}): Promise<{ text: string; events: CapturedAgentEvent[]; compactionCount: number }> { const { extractPayloadText } = await import("./test-helpers.agent-results.js"); const { onAgentEvent } = await import("../infra/agent-events.js"); const events: CapturedAgentEvent[] = []; - const eventPrefix = params.eventPrefix ?? "codex_app_server.guardian"; + const eventPrefixes = params.eventPrefixes ?? [params.eventPrefix ?? "codex_app_server.guardian"]; const unsubscribe = onAgentEvent((event) => { if ( - !event.stream.startsWith(eventPrefix) || + !eventPrefixes.some((prefix) => event.stream.startsWith(prefix)) || (!params.includeAllSessions && event.sessionKey && event.sessionKey !== params.sessionKey) ) { return; @@ -493,7 +548,14 @@ async function requestAgentTextWithEvents(params: { if (payload?.status !== "ok" && !acceptedYieldedTimeout) { throw new Error(`agent status=${String(payload?.status)} payload=${JSON.stringify(payload)}`); } - return { text: extractPayloadText(payload.result), events }; + const result = payload.result as + | { meta?: { agentMeta?: { compactionCount?: number } } } + | undefined; + return { + text: extractPayloadText(payload.result), + events, + compactionCount: Math.max(0, result?.meta?.agentMeta?.compactionCount ?? 0), + }; } finally { unsubscribe(); } @@ -501,7 +563,7 @@ async function requestAgentTextWithEvents(params: { async function requestAgentText(params: { client: GatewayClient; - expectedToken: string; + expectedReply: string; message: string; sessionKey: string; }): Promise { @@ -511,7 +573,16 @@ async function requestAgentText(params: { message: params.message, sessionKey: params.sessionKey, }); - expect(text).toContain(params.expectedToken); + expect(text).toContain(params.expectedReply); + recordCodexAttemptIdentity({ events, sessionKey: params.sessionKey }); + return text; +} + +function recordCodexAttemptIdentity(params: { + events: CapturedAgentEvent[]; + sessionKey: string; +}): void { + const { events } = params; const turnStarting = events.find( (event) => event.stream === "codex_app_server.lifecycle" && event.data?.phase === "turn_starting", @@ -539,7 +610,15 @@ async function requestAgentText(params: { `expected Codex thread_ready identity for ${params.sessionKey}; events=${JSON.stringify(events)}`, ).toBe(true); observedCodexThreadIds.set(params.sessionKey, threadId as string); - return text; + const clientId = threadReady?.data?.clientId; + expect( + typeof clientId === "string" && clientId.trim().length > 0, + `expected Codex client identity for ${params.sessionKey}; events=${JSON.stringify(events)}`, + ).toBe(true); + observedCodexClientIds.set(params.sessionKey, clientId as string); + const action = threadReady?.data?.action; + expect(["started", "resumed", "forked"]).toContain(action); + observedCodexThreadActions.set(params.sessionKey, action as string); } async function verifyCodexCodeModeOnlyDynamicToolProbe(params: { @@ -726,6 +805,127 @@ function formatAssistantTextPreview(texts: string[], maxChars = 800): string { return combined.length > maxChars ? `${combined.slice(0, maxChars)}...` : combined; } +async function readCodexHarnessCompactionCount(params: { + client: GatewayClient; + events: EventFrame[]; + minimum: number; + sessionKey: string; +}): Promise { + const statusText = await requestCodexCommandText({ + client: params.client, + command: "/status", + events: params.events, + expectedText: "Compactions:", + sessionKey: params.sessionKey, + }); + const count = Number(/Compactions:\s*(\d+)/u.exec(statusText)?.[1]); + expect( + count, + `session ${params.sessionKey} did not report ${params.minimum} persisted compactions`, + ).toBeGreaterThanOrEqual(params.minimum); + return count; +} + +async function verifyCodexCompactionStress(params: { + client: GatewayClient; + events: EventFrame[]; + sessionKey: string; +}): Promise<{ hiddenMarker: string; persistedCount: number }> { + const hiddenMarker = `CODEX-DURABLE-${randomBytes(6).toString("hex").toUpperCase()}`; + await requestAgentText({ + client: params.client, + sessionKey: params.sessionKey, + expectedReply: hiddenMarker, + message: [ + `Remember this as durable slot A: ${hiddenMarker}`, + `Reply exactly ${hiddenMarker} and nothing else.`, + ].join("\n"), + }); + const baselineCount = await readCodexHarnessCompactionCount({ + client: params.client, + events: params.events, + minimum: 0, + sessionKey: params.sessionKey, + }); + + const outputLines = Math.ceil(CODEX_HARNESS_LARGE_OUTPUT_BYTES / 90); + let completedCompactions = 0; + let reportedCompactions = 0; + for (let turn = 1; turn <= CODEX_HARNESS_COMPACTION_STRESS_TURNS; turn += 1) { + const acknowledgement = `CODEX-LARGE-OUTPUT-${turn}-${randomBytes(3) + .toString("hex") + .toUpperCase()}`; + const { text, events, compactionCount } = await requestAgentTextWithEvents({ + client: params.client, + eventPrefixes: ["codex_app_server.", "compaction"], + sessionKey: params.sessionKey, + message: [ + "Large-output compaction probe.", + "Use the native exec_command tool exactly once.", + `Run this exact command: node -e 'for(let i=0;i<${outputLines};i++){console.log(i.toString(36).padStart(8,"0")+"-"+((i*2654435761)>>>0).toString(16).padStart(8,"0")+"-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")}'`, + "Set max_output_tokens to 10000.", + `After the tool completes, reply exactly ${acknowledgement} and nothing else.`, + ].join("\n"), + }); + expect(text).toContain(acknowledgement); + recordCodexAttemptIdentity({ events, sessionKey: params.sessionKey }); + completedCompactions += events.filter( + (event) => + event.stream === "compaction" && + event.data?.phase === "end" && + event.data?.completed === true, + ).length; + reportedCompactions += compactionCount; + + const history: { messages?: unknown[] } = await params.client.request("chat.history", { + sessionKey: params.sessionKey, + limit: 100, + }); + const serialized = JSON.stringify(history.messages ?? []); + const originalLengths = Array.from(serialized.matchAll(/original (\d+) chars/gu), (match) => + Number(match[1]), + ); + expect( + originalLengths.some((length) => length > 10_000), + `expected a truncated large native tool result; lengths=${JSON.stringify(originalLengths)}`, + ).toBe(true); + } + + expect(completedCompactions, "expected at least one native automatic compaction").toBeGreaterThan( + 0, + ); + expect(reportedCompactions, "agent result dropped native automatic compactions").toBe( + completedCompactions, + ); + // `/status` stops in the local command handler (`shouldContinue: false`), so + // these snapshots cannot introduce an unobserved native Codex compaction. + const persistedCount = await readCodexHarnessCompactionCount({ + client: params.client, + events: params.events, + minimum: baselineCount + completedCompactions, + sessionKey: params.sessionKey, + }); + expect( + persistedCount - baselineCount, + "persisted session count did not match this wave's native compactions", + ).toBe(completedCompactions); + const recalled = await requestAgentText({ + client: params.client, + sessionKey: params.sessionKey, + expectedReply: hiddenMarker, + message: "Reply with exactly the value stored in durable slot A and nothing else.", + }); + expect(recalled.trim()).toBe(hiddenMarker); + logCodexLiveStep("compaction-stress:complete", { + baselineCount, + completedCompactions, + outputBytes: CODEX_HARNESS_LARGE_OUTPUT_BYTES, + outputTurns: CODEX_HARNESS_COMPACTION_STRESS_TURNS, + persistedCount, + }); + return { hiddenMarker, persistedCount }; +} + async function waitForAssistantText(params: { client: GatewayClient; sessionKey: string; @@ -930,7 +1130,6 @@ async function verifyCodexGuardianProbe(params: { const allowReview = assertGuardianReviewCompleted({ events: allowResult.events, label: "allow probe", - requireEvents: false, }); const allowStatus = findGuardianReviewStatus(allowResult.events); if (allowStatus === "denied") { @@ -977,7 +1176,6 @@ async function verifyCodexGuardianProbe(params: { const review = assertGuardianReviewCompleted({ events: deniedResult.events, label: "ask-back probe", - requireEvents: false, }); // The approve/deny call is Codex policy-owned and may change independently. // OpenClaw's strict projection contract is covered by the allow probe above. @@ -1067,24 +1265,21 @@ async function verifyCodexCronMcpProbe(params: { } } -function hasCodexAppServerLifecycleEvent(params: { - childSessionKey: string; - events: CapturedAgentEvent[]; -}): boolean { - return params.events.some( - (event) => - event.sessionKey === params.childSessionKey && event.stream === "codex_app_server.lifecycle", - ); -} - async function waitForCodexSubagentStarted(params: { childSessionKey: string; events: CapturedAgentEvent[]; -}): Promise { +}): Promise { const deadline = Date.now() + CODEX_HARNESS_REQUEST_TIMEOUT_MS; while (Date.now() < deadline) { - if (hasCodexAppServerLifecycleEvent(params)) { - return; + const threadReady = params.events.find( + (event) => + event.sessionKey === params.childSessionKey && + event.stream === "codex_app_server.lifecycle" && + event.data?.phase === "thread_ready" && + typeof event.data.threadId === "string", + ); + if (threadReady && typeof threadReady.data?.threadId === "string") { + return threadReady.data.threadId; } await delay(2_000); } @@ -1158,36 +1353,84 @@ async function verifyCodexSubagentProbe(params: { } }, }); - const spawnResult = await spawnSubagentDirect( - { - task: `Reply exactly ${expectedToken} and nothing else.`, - agentId: "dev", - thinking: CODEX_HARNESS_THINKING, - mode: "run", - cleanup: "keep", - context: "isolated", - lightContext: true, - expectsCompletionMessage: false, - runTimeoutSeconds: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS, - }, - { - agentSessionKey: params.sessionKey, - }, + const probeReplies = [ + expectedToken, + ...Array.from( + { length: CODEX_HARNESS_SUBAGENT_COUNT - 1 }, + (_, index) => `CODEX-SUBAGENT-${index + 2}-${randomUUID().slice(0, 6).toUpperCase()}`, + ), + ]; + const probes = probeReplies.map((expectedReply, index) => ({ expectedReply, index })); + const spawnResults = await Promise.all( + probes.map(async (probe) => ({ + expectedReply: probe.expectedReply, + index: probe.index, + result: await spawnSubagentDirect( + { + task: `Reply exactly ${probe.expectedReply} and nothing else.`, + agentId: "dev", + thinking: CODEX_HARNESS_THINKING, + mode: "run", + cleanup: "keep", + context: "isolated", + lightContext: true, + expectsCompletionMessage: false, + runTimeoutSeconds: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS, + }, + { + agentSessionKey: params.sessionKey, + }, + ), + })), ); - if (spawnResult.status !== "accepted") { - throw new Error( - `Codex subagent spawn failed: ${JSON.stringify(spawnResult)} trace=${JSON.stringify(gatewayTrace)}`, - ); - } - const childSessionKey = spawnResult.childSessionKey; - if (!childSessionKey?.includes(":subagent:")) { - throw new Error( - `subagent spawn did not return a child session key: ${JSON.stringify(spawnResult)}`, - ); + for (const probe of spawnResults) { + if (probe.result.status !== "accepted") { + throw new Error( + `Codex subagent ${probe.index + 1} spawn failed: ${JSON.stringify(probe.result)} trace=${JSON.stringify(gatewayTrace)}`, + ); + } + if (!probe.result.childSessionKey?.includes(":subagent:") || !probe.result.runId) { + throw new Error( + `subagent spawn did not return child/run identities: ${JSON.stringify(probe.result)}`, + ); + } } - await waitForCodexSubagentStarted({ - childSessionKey, - events, + expect(new Set(spawnResults.map((probe) => probe.result.childSessionKey)).size).toBe( + spawnResults.length, + ); + expect(new Set(spawnResults.map((probe) => probe.result.runId)).size).toBe(spawnResults.length); + + const threadIds = await Promise.all( + spawnResults.map((probe) => + waitForCodexSubagentStarted({ + childSessionKey: probe.result.childSessionKey as string, + events, + }), + ), + ); + expect(new Set(threadIds).size).toBe(threadIds.length); + + await Promise.all( + spawnResults.map((probe) => + waitForChatAgentRunOk(params.client, probe.result.runId as string), + ), + ); + await Promise.all( + spawnResults.map(async (probe) => { + const history: { messages?: unknown[] } = await params.client.request("chat.history", { + sessionKey: probe.result.childSessionKey, + limit: 20, + }); + const replies = extractAssistantTexts(history.messages ?? []); + expect( + replies.some((text) => text.trim() === probe.expectedReply), + `subagent ${probe.index + 1} missing exact reply ${probe.expectedReply}; replies=${formatAssistantTextPreview(replies)}`, + ).toBe(true); + }), + ); + logCodexLiveStep("subagent-fanout:complete", { + count: spawnResults.length, + uniqueThreads: new Set(threadIds).size, }); } finally { const { testing: subagentSpawnTesting } = @@ -1306,6 +1549,7 @@ describeLive("gateway live (Codex harness)", () => { workspace, codexAppServerMode: CODEX_HARNESS_GUARDIAN_PROBE ? "guardian" : "yolo", codeModeOnly: CODEX_HARNESS_CODE_MODE_ONLY, + compactionStress: CODEX_HARNESS_COMPACTION_STRESS, ...(CODEX_HARNESS_DISABLE_LOOP_RELAY ? { loopDetectionPreToolUseRelay: false } : {}), }); const deviceIdentity = await ensurePairedTestGatewayClientIdentity({ @@ -1318,7 +1562,14 @@ describeLive("gateway live (Codex harness)", () => { let guardianPluginApprovalDecision: GuardianPluginApprovalDecision | undefined; let activeApprovalClient: GatewayClient | undefined; let resumeStressState: - | { lastMarker: string; sessionKey: string; threadId: string } + | { + clientId: string; + hiddenMarker?: string; + lastMarker: string; + persistedCompactionCount: number; + sessionKey: string; + threadId: string; + } | undefined; const maybeResolveGuardianPluginApproval = (event: EventFrame): void => { const decision = guardianPluginApprovalDecision; @@ -1345,6 +1596,8 @@ describeLive("gateway live (Codex harness)", () => { }; logCodexLiveStep("config-written", { configPath, modelKey, port }); observedCodexThreadIds.clear(); + observedCodexClientIds.clear(); + observedCodexThreadActions.clear(); try { server = await startGatewayServer(port, { @@ -1411,7 +1664,7 @@ describeLive("gateway live (Codex harness)", () => { const firstText = await requestAgentText({ client: activeClient, sessionKey, - expectedToken: firstToken, + expectedReply: firstToken, message: `Reply with exactly ${firstToken} and nothing else.`, }); expect(firstText).toContain(firstToken); @@ -1422,7 +1675,7 @@ describeLive("gateway live (Codex harness)", () => { const secondText = await requestAgentText({ client: activeClient, sessionKey, - expectedToken: secondToken, + expectedReply: secondToken, message: `Reply with exactly ${secondToken} and nothing else. Do not repeat ${firstToken}.`, }); expect(secondText).toContain(secondToken); @@ -1529,31 +1782,51 @@ describeLive("gateway live (Codex harness)", () => { }); logCodexLiveStep("guardian-probe:done"); } - if (CODEX_HARNESS_RESUME_STRESS) { + const compactionStressState = CODEX_HARNESS_COMPACTION_STRESS + ? await verifyCodexCompactionStress({ + client: activeClient, + events: gatewayEvents, + sessionKey, + }) + : undefined; + if (CODEX_HARNESS_RESTART_STRESS) { const threadId = observedCodexThreadIds.get(sessionKey); if (!threadId) { throw new Error("Codex resume stress did not observe a thread identity"); } + const clientId = observedCodexClientIds.get(sessionKey); + if (!clientId) { + throw new Error("Codex resume stress did not observe a client identity"); + } let lastMarker = ""; - for ( - let historyTurn = 1; - historyTurn <= CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS; - historyTurn += 1 - ) { + const historyTurns = CODEX_HARNESS_RESUME_STRESS + ? CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS + : 1; + for (let historyTurn = 1; historyTurn <= historyTurns; historyTurn += 1) { lastMarker = `CODEX-HISTORY-${historyTurn}-${randomBytes(3) .toString("hex") .toUpperCase()}`; await requestAgentText({ client: activeClient, sessionKey, - expectedToken: lastMarker, - message: `Reply with exactly ${lastMarker} and nothing else.`, + expectedReply: lastMarker, + message: [ + `Replace durable resume slot B with ${lastMarker}.`, + `Reply with exactly ${lastMarker} and nothing else.`, + ].join(" "), }); expect(observedCodexThreadIds.get(sessionKey)).toBe(threadId); } - resumeStressState = { lastMarker, sessionKey, threadId }; + resumeStressState = { + clientId, + hiddenMarker: compactionStressState?.hiddenMarker, + lastMarker, + persistedCompactionCount: compactionStressState?.persistedCount ?? 0, + sessionKey, + threadId, + }; logCodexLiveStep("resume-stress:history-ready", { - historyTurns: CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS + 2, + historyTurns: historyTurns + 2, threadId, }); } @@ -1583,7 +1856,7 @@ describeLive("gateway live (Codex harness)", () => { } } } - if (CODEX_HARNESS_RESUME_STRESS) { + if (CODEX_HARNESS_RESTART_STRESS) { if (!resumeStressState) { throw new Error("Codex resume stress did not seed a thread"); } @@ -1622,25 +1895,67 @@ describeLive("gateway live (Codex harness)", () => { const nextMarker = `CODEX-RESTART-${restart}-${randomBytes(3) .toString("hex") .toUpperCase()}`; + const priorClientId = resumeStressState.clientId; + const replyInstructions = [ + "Read durable resume slot B before replacing it.", + "Reply with exactly the requested lines and nothing else.", + "First line: the current value of durable resume slot B.", + `Second line: ${nextMarker}`, + `Then replace durable resume slot B with ${nextMarker}.`, + ...(resumeStressState.hiddenMarker + ? ["Third line: the value stored in durable slot A."] + : []), + ]; const resumedText = await requestAgentText({ client, sessionKey: resumeStressState.sessionKey, - expectedToken: nextMarker, - message: [ - "Reply with exactly two lines and nothing else.", - "First line: the final marker from the immediately preceding assistant message.", - `Second line: ${nextMarker}`, - ].join(" "), + expectedReply: nextMarker, + message: replyInstructions.join(" "), }); - expect(resumedText.trim()).toBe(`${resumeStressState.lastMarker}\n${nextMarker}`); + const expectedResumeText = [ + resumeStressState.lastMarker, + nextMarker, + ...(resumeStressState.hiddenMarker ? [resumeStressState.hiddenMarker] : []), + ].join("\n"); + expect(resumedText.trim()).toBe(expectedResumeText); const resumedThreadId = observedCodexThreadIds.get(resumeStressState.sessionKey); expect(resumedThreadId).toBe(resumeStressState.threadId); + expect(observedCodexThreadActions.get(resumeStressState.sessionKey)).toBe("resumed"); + const resumedClientId = observedCodexClientIds.get(resumeStressState.sessionKey); + expect(resumedClientId).toBeTruthy(); + expect(resumedClientId).not.toBe(priorClientId); + resumeStressState.clientId = resumedClientId as string; resumeStressState.lastMarker = nextMarker; + if (resumeStressState.persistedCompactionCount > 0) { + await readCodexHarnessCompactionCount({ + client, + events: gatewayEvents, + minimum: resumeStressState.persistedCompactionCount, + sessionKey: resumeStressState.sessionKey, + }); + } logCodexLiveStep("resume-stress:restart-complete", { restart, + action: observedCodexThreadActions.get(resumeStressState.sessionKey), threadId: resumedThreadId, }); } + if (CODEX_HARNESS_SUBAGENT_PROBE && !CODEX_HARNESS_SUBAGENT_ONLY) { + await verifyCodexSubagentProbe({ + client, + sessionKey: resumeStressState.sessionKey, + }); + } + if (CODEX_HARNESS_COMPACTION_STRESS) { + const continued = await verifyCodexCompactionStress({ + client, + events: gatewayEvents, + sessionKey: resumeStressState.sessionKey, + }); + expect(continued.persistedCount).toBeGreaterThan( + resumeStressState.persistedCompactionCount, + ); + } } } finally { try { diff --git a/test/scripts/test-live-codex-harness-docker.test.ts b/test/scripts/test-live-codex-harness-docker.test.ts index 9e3a45758ed..f56f6101abd 100644 --- a/test/scripts/test-live-codex-harness-docker.test.ts +++ b/test/scripts/test-live-codex-harness-docker.test.ts @@ -110,6 +110,24 @@ describe("scripts/test-live-codex-harness-docker.sh", () => { expect(script).toContain( '-e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}"', ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES="${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY="${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}"', + ); }); it("installs the plugin-pinned Codex CLI package for app-server proof", () => { From e721508ed38edeb3e48f7297f1060c4eee6f3dc8 Mon Sep 17 00:00:00 2001 From: "thomas.szbay" Date: Sat, 18 Jul 2026 17:38:12 +0800 Subject: [PATCH 05/76] fix(ui): bound chat avatar fetches (#110010) Co-authored-by: Peter Steinberger --- ui/src/pages/chat/chat-avatar.test.ts | 96 ++++++++++++++++++++++++++- ui/src/pages/chat/chat-avatar.ts | 37 ++++++++++- 2 files changed, 129 insertions(+), 4 deletions(-) diff --git a/ui/src/pages/chat/chat-avatar.test.ts b/ui/src/pages/chat/chat-avatar.test.ts index a19dd1d0c9c..7e4dffd0cc3 100644 --- a/ui/src/pages/chat/chat-avatar.test.ts +++ b/ui/src/pages/chat/chat-avatar.test.ts @@ -1,8 +1,8 @@ /* @vitest-environment jsdom */ import { render } from "lit"; -import { describe, expect, it } from "vitest"; -import { renderChatAvatar } from "./chat-avatar.ts"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { refreshChatAvatar, renderChatAvatar } from "./chat-avatar.ts"; function renderAvatar(params: Parameters) { const container = document.createElement("div"); @@ -10,6 +10,37 @@ function renderAvatar(params: Parameters) { return container.querySelector(".chat-avatar"); } +function createHost(): Parameters[0] { + return { + basePath: "", + chatAvatarUrl: null, + connected: true, + hello: null, + sessionKey: "agent:main", + }; +} + +function pendingUntilAbort(signal: AbortSignal | null | undefined): Promise { + if (!signal) { + throw new Error("expected avatar fetch signal"); + } + return new Promise((_resolve, reject) => { + signal.addEventListener( + "abort", + () => { + const reason = signal.reason; + reject(reason instanceof Error ? reason : new Error("avatar fetch aborted")); + }, + { once: true }, + ); + }); +} + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + describe("renderChatAvatar", () => { it("renders assistant fallback, blob image, and text avatars", () => { const defaultAvatar = renderAvatar(["assistant"]); @@ -53,3 +84,64 @@ describe("renderChatAvatar", () => { expect(textAvatar?.textContent?.trim()).toBe("AB"); }); }); + +describe("refreshChatAvatar", () => { + it("aborts a stalled metadata fetch at the deadline", async () => { + vi.useFakeTimers(); + const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => + pendingUntilAbort(init?.signal), + ); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + + const host = createHost(); + const refresh = refreshChatAvatar(host); + const signal = fetchMock.mock.calls[0]?.[1]?.signal; + expect(signal?.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(29_999); + expect(signal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(signal?.aborted).toBe(true); + + await expect(refresh).resolves.toBeUndefined(); + expect(host.chatAvatarUrl).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("keeps the image body read bounded by its own deadline", async () => { + vi.useFakeTimers(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ avatarUrl: "/avatar/main" }), + }) + .mockImplementationOnce((_input: RequestInfo | URL, init?: RequestInit) => + Promise.resolve({ + ok: true, + blob: () => pendingUntilAbort(init?.signal), + }), + ); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + + const host = createHost(); + const refresh = refreshChatAvatar(host); + await vi.advanceTimersByTimeAsync(0); + + expect(fetchMock).toHaveBeenCalledTimes(2); + const metadataSignal = fetchMock.mock.calls[0]?.[1]?.signal as AbortSignal | undefined; + const imageSignal = fetchMock.mock.calls[1]?.[1]?.signal as AbortSignal | undefined; + expect(metadataSignal).not.toBe(imageSignal); + expect(metadataSignal?.aborted).toBe(false); + expect(imageSignal?.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(29_999); + expect(imageSignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(imageSignal?.aborted).toBe(true); + + await expect(refresh).resolves.toBeUndefined(); + expect(host.chatAvatarUrl).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/ui/src/pages/chat/chat-avatar.ts b/ui/src/pages/chat/chat-avatar.ts index 41063cd4a99..592736d2399 100644 --- a/ui/src/pages/chat/chat-avatar.ts +++ b/ui/src/pages/chat/chat-avatar.ts @@ -262,6 +262,16 @@ function isLocalControlUiAvatarUrl(avatarUrl: string): boolean { return avatarUrl.startsWith("/"); } +/** Give each sequential fetch a full budget; sharing one can starve the image request. */ +const CHAT_AVATAR_FETCH_TIMEOUT_MS = 30_000; + +function scheduleChatAvatarFetchTimeout(controller: AbortController, label: string) { + return setTimeout( + () => controller.abort(new DOMException(`${label} timed out`, "TimeoutError")), + CHAT_AVATAR_FETCH_TIMEOUT_MS, + ); +} + export async function refreshChatAvatar(host: ChatAvatarHost) { if (!host.connected) { clearChatAvatarState(host); @@ -280,8 +290,16 @@ export async function refreshChatAvatar(host: ChatAvatarHost) { const authHeader = resolveControlUiAuthHeader(host); const headers = buildControlUiAuthHeaders(authHeader); const url = buildAvatarMetaUrl(host.basePath, agentId); + + const metaController = new AbortController(); + const metaTimeout = scheduleChatAvatarFetchTimeout(metaController, "chat avatar metadata fetch"); + let avatarUrl: string; try { - const res = await fetch(url, { method: "GET", ...(headers ? { headers } : {}) }); + const res = await fetch(url, { + method: "GET", + ...(headers ? { headers } : {}), + signal: metaController.signal, + }); if (!shouldApplyChatAvatarResult(host, requestVersion, sessionKey, agentId)) { return; } @@ -299,7 +317,7 @@ export async function refreshChatAvatar(host: ChatAvatarHost) { return; } setChatAvatarMeta(host, data); - const avatarUrl = typeof data.avatarUrl === "string" ? data.avatarUrl.trim() : ""; + avatarUrl = typeof data.avatarUrl === "string" ? data.avatarUrl.trim() : ""; if (!avatarUrl || !isRenderableControlUiAvatarUrl(avatarUrl)) { clearChatAvatarUrl(host); return; @@ -308,9 +326,22 @@ export async function refreshChatAvatar(host: ChatAvatarHost) { setChatAvatarUrl(host, avatarUrl); return; } + } catch { + if (shouldApplyChatAvatarResult(host, requestVersion, sessionKey, agentId)) { + clearChatAvatarState(host); + } + return; + } finally { + clearTimeout(metaTimeout); + } + + const avatarController = new AbortController(); + const avatarTimeout = scheduleChatAvatarFetchTimeout(avatarController, "chat avatar image fetch"); + try { const avatarRes = await fetch(avatarUrl, { method: "GET", ...(headers ? { headers } : {}), + signal: avatarController.signal, }); if (!avatarRes.ok) { if (shouldApplyChatAvatarResult(host, requestVersion, sessionKey, agentId)) { @@ -328,5 +359,7 @@ export async function refreshChatAvatar(host: ChatAvatarHost) { if (shouldApplyChatAvatarResult(host, requestVersion, sessionKey, agentId)) { clearChatAvatarState(host); } + } finally { + clearTimeout(avatarTimeout); } } From 39528edd745a484457e2add67155f611b2f7da5b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 02:38:13 -0700 Subject: [PATCH 06/76] fix(security): restore fs import lost in bounded manifest-read merge #101773 removed the node:fs/promises import while switching plugin manifest reads to fs-safe helpers, but getSkillCodeSafetySummary (added later by #109363) still reads SKILL.md via fs.readFile. Main has been red since the merge (check-prod-types, check-lint, check-test-types, package-boundary compile, compact-large-5). Restore the import; skill-file reads keep their pre-existing unbounded contract. --- .../gateway-protocol/src/schema/nodes.test.ts | 34 ----- packages/gateway-protocol/src/schema/nodes.ts | 17 ++- .../src/schema/session-placement.test.ts | 24 ---- .../src/schema/session-placement.ts | 128 +++++++++++------- 4 files changed, 87 insertions(+), 116 deletions(-) delete mode 100644 packages/gateway-protocol/src/schema/nodes.test.ts diff --git a/packages/gateway-protocol/src/schema/nodes.test.ts b/packages/gateway-protocol/src/schema/nodes.test.ts deleted file mode 100644 index 5735413cc91..00000000000 --- a/packages/gateway-protocol/src/schema/nodes.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { validateNodeInvokeProgressParams } from "../index.js"; - -describe("node protocol schemas", () => { - it("accepts bounded progress chunks and rejects extra fields", () => { - expect( - validateNodeInvokeProgressParams({ - invokeId: "invoke-1", - nodeId: "node-1", - seq: 0, - chunk: "stdout line", - }), - ).toBe(true); - - expect( - validateNodeInvokeProgressParams({ - invokeId: "invoke-1", - nodeId: "node-1", - seq: 0, - chunk: "x".repeat(16 * 1024 + 1), - }), - ).toBe(false); - - expect( - validateNodeInvokeProgressParams({ - invokeId: "invoke-1", - nodeId: "node-1", - seq: 0, - chunk: "stdout line", - extra: "not allowed", - }), - ).toBe(false); - }); -}); diff --git a/packages/gateway-protocol/src/schema/nodes.ts b/packages/gateway-protocol/src/schema/nodes.ts index 47fd1a300ec..40e8ce9fea5 100644 --- a/packages/gateway-protocol/src/schema/nodes.ts +++ b/packages/gateway-protocol/src/schema/nodes.ts @@ -164,13 +164,16 @@ export const NodeInvokeResultParamsSchema = closedObject({ }); /** Ordered UTF-8 output emitted while a node command invocation is running. */ -export const NodeInvokeProgressParamsSchema = closedObject({ - invokeId: NonEmptyString, - nodeId: NonEmptyString, - seq: Type.Integer({ minimum: 0 }), - // Empty chunks are liveness heartbeats for captured stderr or capped stdout. - chunk: Type.String({ maxLength: 16 * 1024 }), -}); +export const NodeInvokeProgressParamsSchema = Type.Object( + { + invokeId: NonEmptyString, + nodeId: NonEmptyString, + seq: Type.Integer({ minimum: 0 }), + // Empty chunks are liveness heartbeats for captured stderr or capped stdout. + chunk: Type.String({ maxLength: 16 * 1024 }), + }, + { additionalProperties: false }, +); /** Generic node event envelope accepted by the gateway. */ export const NodeEventParamsSchema = closedObject({ diff --git a/packages/gateway-protocol/src/schema/session-placement.test.ts b/packages/gateway-protocol/src/schema/session-placement.test.ts index 789b72bb629..8943d06e094 100644 --- a/packages/gateway-protocol/src/schema/session-placement.test.ts +++ b/packages/gateway-protocol/src/schema/session-placement.test.ts @@ -282,28 +282,4 @@ describe("session dispatch protocol schemas", () => { }), ).toBe(false); }); - - it("rejects extra fields in dispatch params and results", () => { - const active = { - state: "active" as const, - ...basePlacement, - ...workerOwnedFields, - }; - expect( - validateSessionsDispatchResult({ - ok: true, - key: "agent:main:dispatch", - sessionId: "session-1", - placement: active, - extra: true, - }), - ).toBe(false); - expect( - validateSessionsDispatchParams({ - key: "agent:main:dispatch", - profileId: "development", - extra: true, - }), - ).toBe(false); - }); }); diff --git a/packages/gateway-protocol/src/schema/session-placement.ts b/packages/gateway-protocol/src/schema/session-placement.ts index 34acbab4ff7..65ff59e7989 100644 --- a/packages/gateway-protocol/src/schema/session-placement.ts +++ b/packages/gateway-protocol/src/schema/session-placement.ts @@ -1,6 +1,5 @@ import type { Static } from "typebox"; import { Type } from "typebox"; -import { closedObject } from "./closed-object.js"; import { NonEmptyString } from "./primitives.js"; import { SESSION_PLACEMENT_STATES } from "./session-placement-state.js"; @@ -78,64 +77,85 @@ const TerminalSessionPlacementProperties = { function createUnownedSessionPlacementSchema( state: State, ) { - return closedObject({ state: Type.Literal(state), ...SessionPlacementTimingProperties }); + return Type.Object( + { state: Type.Literal(state), ...SessionPlacementTimingProperties }, + { additionalProperties: false }, + ); } function createWorkerOwnedSessionPlacementSchema< const State extends "active" | "draining" | "reconciling", >(state: State) { - return closedObject({ - state: Type.Literal(state), - ...SessionPlacementTimingProperties, - environmentId: NonEmptyString, - activeOwnerEpoch: SessionPlacementOwnerEpochSchema, - workerBundleHash: WorkerBundleHashSchema, - ...SessionPlacementWorkspaceProperties, - ...SessionPlacementAckProperties, - }); + return Type.Object( + { + state: Type.Literal(state), + ...SessionPlacementTimingProperties, + environmentId: NonEmptyString, + activeOwnerEpoch: SessionPlacementOwnerEpochSchema, + workerBundleHash: WorkerBundleHashSchema, + ...SessionPlacementWorkspaceProperties, + ...SessionPlacementAckProperties, + }, + { additionalProperties: false }, + ); } export const LocalSessionPlacementSchema = createUnownedSessionPlacementSchema("local"); export const RequestedSessionPlacementSchema = createUnownedSessionPlacementSchema("requested"); -export const ProvisioningSessionPlacementSchema = closedObject({ - state: Type.Literal("provisioning"), - ...SessionPlacementTimingProperties, - environmentId: Type.Optional(NonEmptyString), -}); +export const ProvisioningSessionPlacementSchema = Type.Object( + { + state: Type.Literal("provisioning"), + ...SessionPlacementTimingProperties, + environmentId: Type.Optional(NonEmptyString), + }, + { additionalProperties: false }, +); -export const SyncingSessionPlacementSchema = closedObject({ - state: Type.Literal("syncing"), - ...SessionPlacementTimingProperties, - environmentId: NonEmptyString, - workerBundleHash: WorkerBundleHashSchema, -}); +export const SyncingSessionPlacementSchema = Type.Object( + { + state: Type.Literal("syncing"), + ...SessionPlacementTimingProperties, + environmentId: NonEmptyString, + workerBundleHash: WorkerBundleHashSchema, + }, + { additionalProperties: false }, +); -export const StartingSessionPlacementSchema = closedObject({ - state: Type.Literal("starting"), - ...SessionPlacementTimingProperties, - environmentId: NonEmptyString, - workerBundleHash: WorkerBundleHashSchema, - ...SessionPlacementWorkspaceProperties, -}); +export const StartingSessionPlacementSchema = Type.Object( + { + state: Type.Literal("starting"), + ...SessionPlacementTimingProperties, + environmentId: NonEmptyString, + workerBundleHash: WorkerBundleHashSchema, + ...SessionPlacementWorkspaceProperties, + }, + { additionalProperties: false }, +); export const ActiveWorkerSessionPlacementSchema = createWorkerOwnedSessionPlacementSchema("active"); export const DrainingSessionPlacementSchema = createWorkerOwnedSessionPlacementSchema("draining"); export const ReconcilingSessionPlacementSchema = createWorkerOwnedSessionPlacementSchema("reconciling"); -export const ReclaimedSessionPlacementSchema = closedObject({ - state: Type.Literal("reclaimed"), - ...SessionPlacementTimingProperties, - ...TerminalSessionPlacementProperties, -}); +export const ReclaimedSessionPlacementSchema = Type.Object( + { + state: Type.Literal("reclaimed"), + ...SessionPlacementTimingProperties, + ...TerminalSessionPlacementProperties, + }, + { additionalProperties: false }, +); -export const FailedSessionPlacementSchema = closedObject({ - state: Type.Literal("failed"), - ...SessionPlacementTimingProperties, - ...TerminalSessionPlacementProperties, - recoveryError: NonEmptyString, -}); +export const FailedSessionPlacementSchema = Type.Object( + { + state: Type.Literal("failed"), + ...SessionPlacementTimingProperties, + ...TerminalSessionPlacementProperties, + recoveryError: NonEmptyString, + }, + { additionalProperties: false }, +); /** Gateway-visible placement projection; `state` remains the closed discriminator. */ export const SessionPlacementSchema = Type.Union([ @@ -152,19 +172,25 @@ export const SessionPlacementSchema = Type.Union([ ]); /** Requests one-way dispatch of an existing local session to a configured worker profile. */ -export const SessionsDispatchParamsSchema = closedObject({ - key: NonEmptyString, - agentId: Type.Optional(NonEmptyString), - profileId: NonEmptyString, -}); +export const SessionsDispatchParamsSchema = Type.Object( + { + key: NonEmptyString, + agentId: Type.Optional(NonEmptyString), + profileId: NonEmptyString, + }, + { additionalProperties: false }, +); /** Result returned once session dispatch reaches durable worker ownership. */ -export const SessionsDispatchResultSchema = closedObject({ - ok: Type.Literal(true), - key: NonEmptyString, - sessionId: NonEmptyString, - placement: ActiveWorkerSessionPlacementSchema, -}); +export const SessionsDispatchResultSchema = Type.Object( + { + ok: Type.Literal(true), + key: NonEmptyString, + sessionId: NonEmptyString, + placement: ActiveWorkerSessionPlacementSchema, + }, + { additionalProperties: false }, +); /** Requests safe workspace reconciliation and teardown of an active cloud worker. */ export const SessionsReclaimParamsSchema = Type.Object( From b34d0a8bb6aaac68669e791362083b8772b5f30d Mon Sep 17 00:00:00 2001 From: tzy-17 Date: Sat, 18 Jul 2026 17:40:05 +0800 Subject: [PATCH 07/76] fix(gateway): bound usage.list days parameter to 100 years (#110552) * fix(gateway): bound usage.list days parameter to 100 years parseDays accepted any finite number including 1e300, which downstream resolveTrailingDays would feed into Date arithmetic as a day offset, overflowing the ECMAScript Date range and surfacing as a misleading "calendar day does not exist in requested time zone" error. Reject day counts above 100 years (36600); callers wanting truly unbounded history should use `range: "all"`. Co-authored-by: Patrick Erichsen * fix(gateway): clamp oversized usage day ranges --------- Co-authored-by: Patrick Erichsen Co-authored-by: Peter Steinberger --- src/gateway/server-methods/usage.test.ts | 13 ++++++++++++- src/gateway/server-methods/usage.ts | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/gateway/server-methods/usage.test.ts b/src/gateway/server-methods/usage.test.ts index 1cab7dc5da1..f4ab4ff8086 100644 --- a/src/gateway/server-methods/usage.test.ts +++ b/src/gateway/server-methods/usage.test.ts @@ -266,6 +266,13 @@ describe("gateway usage helpers", () => { expect(testApi.parseDays("nope")).toBeUndefined(); }); + it("parseDays caps day counts before Date arithmetic can overflow", () => { + expect(testApi.parseDays(1e300)).toBe(36600); + expect(testApi.parseDays("1e300")).toBe(36600); + expect(testApi.parseDays(Number.MAX_SAFE_INTEGER)).toBe(36600); + expect(testApi.parseDays(366 * 100)).toBe(36600); + }); + it("resolveDateRange uses explicit start/end as UTC when mode is missing (backward compatible)", () => { const result = testApi.resolveDateRange({ startDate: "2026-02-01", @@ -413,13 +420,17 @@ describe("gateway usage helpers", () => { }); }); - it("resolveDateRange clamps days to at least 1 and defaults to 30 days", () => { + it("resolveDateRange clamps days to its supported bounds and defaults to 30 days", () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-02-05T12:34:56.000Z")); const oneDay = expectDateRange(testApi.resolveDateRange({ days: 0 })); expect(oneDay.endMs).toBe(Date.UTC(2026, 1, 5) + dayMs - 1); expect(oneDay.startMs).toBe(Date.UTC(2026, 1, 5)); + const maxDays = expectDateRange(testApi.resolveDateRange({ days: Number.MAX_SAFE_INTEGER })); + expect(maxDays.endMs).toBe(Date.UTC(2026, 1, 5) + dayMs - 1); + expect(maxDays.startMs).toBe(Date.UTC(2026, 1, 5) - (36600 - 1) * dayMs); + const def = expectDateRange(testApi.resolveDateRange({})); expect(def.endMs).toBe(Date.UTC(2026, 1, 5) + dayMs - 1); expect(def.startMs).toBe(Date.UTC(2026, 1, 5) - 29 * dayMs); diff --git a/src/gateway/server-methods/usage.ts b/src/gateway/server-methods/usage.ts index b9cfca2aaee..655348ba825 100644 --- a/src/gateway/server-methods/usage.ts +++ b/src/gateway/server-methods/usage.ts @@ -88,6 +88,10 @@ type DateRange = { startMs: number; endMs: number; includeUntimestamped?: boolea // Keep validation and parsed timestamps in one result so handlers cannot forward // an invalid or backwards window to the usage loaders. type DateRangeResolution = { ok: true; value: DateRange } | { ok: false; error: string }; +// 100 years: callers requesting unbounded history should use `range: "all"`. +// Larger explicit day counts would overflow ECMAScript Date arithmetic and +// surface as a misleading "calendar day does not exist" error from the resolver. +const MAX_USAGE_DAYS = 366 * 100; type DateInterpretation = | { mode: "utc" | "gateway" } | { mode: "utc-offset"; utcOffsetMinutes: number } @@ -388,14 +392,17 @@ const formatDateParts = (year: number, monthIndex: number, day: number): string `${year}-${String(monthIndex + 1).padStart(2, "0")}-${String(day).padStart(2, "0")}`; const parseDays = (raw: unknown): number | undefined => { - if (typeof raw === "number" && Number.isFinite(raw)) { - return Math.floor(raw); + const fromFinite = (n: number): number | undefined => { + if (!Number.isFinite(n)) { + return undefined; + } + return Math.min(Math.floor(n), MAX_USAGE_DAYS); + }; + if (typeof raw === "number") { + return fromFinite(raw); } if (typeof raw === "string" && raw.trim() !== "") { - const parsed = Number(raw); - if (Number.isFinite(parsed)) { - return Math.floor(parsed); - } + return fromFinite(Number(raw)); } return undefined; }; From 1af6d99658098ceb2f75f36acf68b92c8eefe635 Mon Sep 17 00:00:00 2001 From: xingzhou Date: Sat, 18 Jul 2026 17:41:31 +0800 Subject: [PATCH 08/76] fix: surface cron errors for empty isolated agent replies (#100229) * fix(cron): fail empty isolated agent completions * fix: surface cron errors for empty isolated agent replies * fix(cron): preserve intentional terminal progress --------- Co-authored-by: Peter Steinberger Co-authored-by: Peter Steinberger --- .../run.meta-error-status.test.ts | 274 ++++++++++++++++++ src/cron/isolated-agent/run.ts | 36 +++ 2 files changed, 310 insertions(+) diff --git a/src/cron/isolated-agent/run.meta-error-status.test.ts b/src/cron/isolated-agent/run.meta-error-status.test.ts index c85f5668dc4..57ae4c0d92a 100644 --- a/src/cron/isolated-agent/run.meta-error-status.test.ts +++ b/src/cron/isolated-agent/run.meta-error-status.test.ts @@ -4,7 +4,10 @@ import { makeIsolatedAgentJobFixture, makeIsolatedAgentParamsFixture } from "./j import { setupRunCronIsolatedAgentTurnSuite } from "./run.suite-helpers.js"; import { cleanupDirectCronSessionMock, + dispatchCronDeliveryMock, loadRunCronIsolatedAgentTurn, + resolveCronDeliveryPlanMock, + resolveCronPayloadOutcomeMock, runWithModelFallbackMock, } from "./run.test-harness.js"; @@ -94,6 +97,277 @@ describe("runCronIsolatedAgentTurn - meta.error status propagation", () => { }); }); + it("marks a completed embedded run with no final payload as a cron error", async () => { + runWithModelFallbackMock.mockResolvedValueOnce({ + result: { + payloads: [], + meta: { + agentMeta: { usage: { input: 10, output: 0 } }, + }, + }, + provider: "anthropic", + model: "claude-opus-4-8", + attempts: [], + }); + resolveCronDeliveryPlanMock.mockReturnValue({ + requested: true, + mode: "announce", + channel: "messagechat", + to: "test-target", + }); + resolveCronPayloadOutcomeMock.mockReturnValue({ + summary: undefined, + outputText: undefined, + synthesizedText: undefined, + deliveryPayload: undefined, + deliveryPayloads: [], + deliveryPayloadHasStructuredContent: false, + hasFatalErrorPayload: false, + hasFatalStructuredErrorPayload: false, + embeddedRunError: undefined, + }); + + const result = await runCronIsolatedAgentTurn(makeIsolatedAgentParamsFixture()); + + expect(dispatchCronDeliveryMock).not.toHaveBeenCalled(); + expect(result.status).toBe("error"); + expect(result.error).toBe("cron isolated run completed without a final assistant payload"); + expect(result.delivered).toBe(false); + }); + + it("marks empty message-tool attempts without source delivery as cron errors", async () => { + runWithModelFallbackMock.mockResolvedValueOnce({ + result: { + payloads: [], + didSendViaMessagingTool: true, + messagingToolSentTargets: [], + meta: { + agentMeta: { usage: { input: 10, output: 0 } }, + }, + }, + provider: "anthropic", + model: "claude-opus-4-8", + attempts: [], + }); + resolveCronDeliveryPlanMock.mockReturnValue({ + requested: true, + mode: "announce", + channel: "messagechat", + to: "test-target", + }); + resolveCronPayloadOutcomeMock.mockReturnValue({ + summary: undefined, + outputText: undefined, + synthesizedText: undefined, + deliveryPayload: undefined, + deliveryPayloads: [], + deliveryPayloadHasStructuredContent: false, + hasFatalErrorPayload: false, + hasFatalStructuredErrorPayload: false, + embeddedRunError: undefined, + }); + + const result = await runCronIsolatedAgentTurn(makeIsolatedAgentParamsFixture()); + + expect(dispatchCronDeliveryMock).not.toHaveBeenCalled(); + expect(result.status).toBe("error"); + expect(result.error).toBe("cron isolated run completed without a final assistant payload"); + expect(result.delivered).toBe(false); + }); + + it("keeps explicit silent replies as successful cron completions", async () => { + runWithModelFallbackMock.mockResolvedValueOnce({ + result: { + payloads: [], + meta: { + finalAssistantRawText: "NO_REPLY", + finalAssistantVisibleText: "NO_REPLY", + agentMeta: { usage: { input: 10, output: 1 } }, + }, + }, + provider: "anthropic", + model: "claude-opus-4-8", + attempts: [], + }); + resolveCronDeliveryPlanMock.mockReturnValue({ + requested: true, + mode: "announce", + channel: "messagechat", + to: "test-target", + }); + resolveCronPayloadOutcomeMock.mockReturnValue({ + summary: undefined, + outputText: undefined, + synthesizedText: undefined, + deliveryPayload: undefined, + deliveryPayloads: [], + deliveryPayloadHasStructuredContent: false, + hasFatalErrorPayload: false, + hasFatalStructuredErrorPayload: false, + embeddedRunError: undefined, + }); + + const result = await runCronIsolatedAgentTurn(makeIsolatedAgentParamsFixture()); + + expect(dispatchCronDeliveryMock).toHaveBeenCalled(); + expect(result.status).toBe("ok"); + expect(result.error).toBeUndefined(); + }); + + it("keeps committed message-tool deliveries as successful cron completions", async () => { + runWithModelFallbackMock.mockResolvedValueOnce({ + result: { + payloads: [], + didSendViaMessagingTool: true, + messagingToolSentTexts: ["Delivered to an intentional recipient"], + messagingToolSentTargets: [], + meta: { + agentMeta: { usage: { input: 10, output: 0 } }, + }, + }, + provider: "anthropic", + model: "claude-opus-4-8", + attempts: [], + }); + resolveCronDeliveryPlanMock.mockReturnValue({ + requested: true, + mode: "announce", + channel: "messagechat", + to: "test-target", + }); + resolveCronPayloadOutcomeMock.mockReturnValue({ + summary: undefined, + outputText: undefined, + synthesizedText: undefined, + deliveryPayload: undefined, + deliveryPayloads: [], + deliveryPayloadHasStructuredContent: false, + hasFatalErrorPayload: false, + hasFatalStructuredErrorPayload: false, + embeddedRunError: undefined, + }); + + const result = await runCronIsolatedAgentTurn(makeIsolatedAgentParamsFixture()); + + expect(dispatchCronDeliveryMock).toHaveBeenCalled(); + expect(result.status).toBe("ok"); + expect(result.error).toBeUndefined(); + }); + + it("does not mark empty deterministic approval prompts as cron errors", async () => { + runWithModelFallbackMock.mockResolvedValueOnce({ + result: { + payloads: [], + didSendDeterministicApprovalPrompt: true, + meta: { + agentMeta: { usage: { input: 10, output: 0 } }, + }, + }, + provider: "anthropic", + model: "claude-opus-4-8", + attempts: [], + }); + resolveCronDeliveryPlanMock.mockReturnValue({ + requested: true, + mode: "announce", + channel: "messagechat", + to: "test-target", + }); + resolveCronPayloadOutcomeMock.mockReturnValue({ + summary: undefined, + outputText: undefined, + synthesizedText: undefined, + deliveryPayload: undefined, + deliveryPayloads: [], + deliveryPayloadHasStructuredContent: false, + hasFatalErrorPayload: false, + hasFatalStructuredErrorPayload: false, + embeddedRunError: undefined, + }); + + const result = await runCronIsolatedAgentTurn(makeIsolatedAgentParamsFixture()); + + expect(dispatchCronDeliveryMock).toHaveBeenCalled(); + expect(result.status).toBe("ok"); + expect(result.error).toBeUndefined(); + }); + + it("does not mark empty accepted child-session handoffs as cron errors", async () => { + runWithModelFallbackMock.mockResolvedValueOnce({ + result: { + payloads: [], + acceptedSessionSpawns: [{ runId: "run-child", childSessionKey: "agent:default:child" }], + meta: { + agentMeta: { usage: { input: 10, output: 0 } }, + }, + }, + provider: "anthropic", + model: "claude-opus-4-8", + attempts: [], + }); + resolveCronDeliveryPlanMock.mockReturnValue({ + requested: true, + mode: "announce", + channel: "messagechat", + to: "test-target", + }); + resolveCronPayloadOutcomeMock.mockReturnValue({ + summary: undefined, + outputText: undefined, + synthesizedText: undefined, + deliveryPayload: undefined, + deliveryPayloads: [], + deliveryPayloadHasStructuredContent: false, + hasFatalErrorPayload: false, + hasFatalStructuredErrorPayload: false, + embeddedRunError: undefined, + }); + + const result = await runCronIsolatedAgentTurn(makeIsolatedAgentParamsFixture()); + + expect(dispatchCronDeliveryMock).toHaveBeenCalled(); + expect(result.status).toBe("ok"); + expect(result.error).toBeUndefined(); + }); + + it("does not mark empty successful cron-add completions as cron errors", async () => { + runWithModelFallbackMock.mockResolvedValueOnce({ + result: { + payloads: [], + successfulCronAdds: 1, + meta: { + agentMeta: { usage: { input: 10, output: 0 } }, + }, + }, + provider: "anthropic", + model: "claude-opus-4-8", + attempts: [], + }); + resolveCronDeliveryPlanMock.mockReturnValue({ + requested: true, + mode: "announce", + channel: "messagechat", + to: "test-target", + }); + resolveCronPayloadOutcomeMock.mockReturnValue({ + summary: undefined, + outputText: undefined, + synthesizedText: undefined, + deliveryPayload: undefined, + deliveryPayloads: [], + deliveryPayloadHasStructuredContent: false, + hasFatalErrorPayload: false, + hasFatalStructuredErrorPayload: false, + embeddedRunError: undefined, + }); + + const result = await runCronIsolatedAgentTurn(makeIsolatedAgentParamsFixture()); + + expect(dispatchCronDeliveryMock).toHaveBeenCalled(); + expect(result.status).toBe("ok"); + expect(result.error).toBeUndefined(); + }); + it("surfaces cron timeout result when the cron-nested lane watchdog fires", async () => { runWithModelFallbackMock.mockRejectedValueOnce( makeCommandLaneTaskTimeoutError("cron-nested", 330_000), diff --git a/src/cron/isolated-agent/run.ts b/src/cron/isolated-agent/run.ts index 7d556884a38..e35d087f56e 100644 --- a/src/cron/isolated-agent/run.ts +++ b/src/cron/isolated-agent/run.ts @@ -1,7 +1,9 @@ import { isDeepStrictEqual } from "node:util"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { hasAcceptedSessionSpawn } from "../../agents/accepted-session-spawn.js"; import { retireSessionMcpRuntime } from "../../agents/agent-bundle-mcp-tools.js"; import { hasAnyAuthProfileStoreSource } from "../../agents/auth-profiles/source-check.js"; +import { hasCommittedMessagingToolDeliveryEvidence } from "../../agents/embedded-agent-runner/delivery-evidence.js"; import { findModelInCatalog } from "../../agents/model-catalog-lookup.js"; import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js"; import { listOpenAIAuthProfileProvidersForAgentRuntime } from "../../agents/openai-routing.js"; @@ -9,6 +11,7 @@ import { createAgentRunRestartAbortError } from "../../agents/run-termination.js import { expandToolGroups, normalizeToolName } from "../../agents/tool-policy.js"; import { deriveContextPromptTokens } from "../../agents/usage.js"; import type { ThinkLevel } from "../../auto-reply/thinking.js"; +import { isSilentReplyPayloadText } from "../../auto-reply/tokens.js"; import type { CliDeps } from "../../cli/outbound-send-deps.js"; import { resolveAgentModelPrimaryValue } from "../../config/model-input.js"; import type { SessionEntry } from "../../config/sessions.js"; @@ -1442,6 +1445,39 @@ async function finalizeCronRun(params: { sourceDeliveryOutcome, }); } + const hasCommittedTerminalProgress = + hasCommittedMessagingToolDeliveryEvidence(finalRunResult) || + finalRunResult.didSendDeterministicApprovalPrompt === true || + hasAcceptedSessionSpawn(finalRunResult.acceptedSessionSpawns) || + (finalRunResult.successfulCronAdds ?? 0) > 0; + const hasIntentionalSilentReply = + finalRunResult.meta?.terminalReplyKind === "silent-empty" || + isSilentReplyPayloadText(finalRunResult.meta?.finalAssistantRawText) || + isSilentReplyPayloadText(finalRunResult.meta?.finalAssistantVisibleText); + if ( + prepared.deliveryRequested && + !hasFatalErrorPayload && + !sourceDeliveryOutcome.satisfiesSourceDelivery && + !hasCommittedTerminalProgress && + !hasIntentionalSilentReply && + deliveryPayloads.length === 0 && + normalizeOptionalString(synthesizedText) === undefined + ) { + const error = "cron isolated run completed without a final assistant payload"; + return prepared.withRunSession({ + status: "error", + error, + summary: error, + outputText: error, + delivered: false, + deliveryAttempted: false, + diagnostics: mergeCronRunDiagnostics( + runDiagnostics, + createCronRunDiagnosticsFromError("agent-run", error), + ), + ...telemetry, + }); + } if (hasFatalStructuredErrorPayload && prepared.deliveryRequested) { // Structured run error payloads belong in cron state and failure alerts, // not the normal completion announce path where provider JSON can leak. From 2b04c350a322a038fe6677c848cf654c80d9f74c Mon Sep 17 00:00:00 2001 From: Peter Lee Date: Sat, 18 Jul 2026 04:41:44 -0500 Subject: [PATCH 09/76] fix(config): restore process.env after invalid config rejection (#103786) * fix(config): restore process.env after invalid config rejection When config validation fails, env vars from the rejected config (env.vars in openclaw.json) leaked into process.env and were never cleaned up. In long-running gateways this meant rejected config values could persist until process restart. Add restoreEnvChangesIfUnchanged calls in both sync (loadConfigLocal) and async (readConfigFileSnapshotInternal) validation-failure paths, matching the pattern already used by suspicious-config recovery paths. * fix(config): restore env before observer and on duplicate-agent-dir rejection - Add restoreEnvChangesIfUnchanged before DuplicateAgentDirError throw in the sync loadConfigLocal path so env vars from rejected config do not leak when duplicate agent directories are detected. - Move restoreEnvChangesIfUnchanged before observeLoadConfigSnapshot in the sync validation-failure path so config health/audit observation does not see rejected env values. - Add entry-point regression tests exercising sync loadConfig, async readConfigFileSnapshot, and duplicate-agent-dir rejection paths through createConfigIO with real config files. * fix(config): restore env after invalid snapshot Co-authored-by: xialonglee --------- Co-authored-by: Peter Steinberger --- src/config/io.load-env-restore.test.ts | 47 ++++++++++++++++++++++++++ src/config/io.snapshot.ts | 6 ++++ 2 files changed, 53 insertions(+) diff --git a/src/config/io.load-env-restore.test.ts b/src/config/io.load-env-restore.test.ts index 6f5be4b868f..f252371d991 100644 --- a/src/config/io.load-env-restore.test.ts +++ b/src/config/io.load-env-restore.test.ts @@ -123,3 +123,50 @@ describe("loadConfig env restoration", () => { }); }); }); + +describe("readConfigFileSnapshot env restoration", () => { + it("removes a newly injected env var after invalid snapshot validation", async () => { + await withTempHome(async (home) => { + await writeOpenClawConfig(home, { + env: { vars: { TEST_VAR: "injected-value" } }, + gateway: { port: "invalid" }, + }); + + const env = { HOME: home } as NodeJS.ProcessEnv; + const io = createConfigIO({ + env, + homedir: () => home, + logger: { warn: () => {}, error: () => {} }, + }); + + const snapshot = await io.readConfigFileSnapshot(); + + expect(snapshot.valid).toBe(false); + expect(env.TEST_VAR).toBeUndefined(); + }); + }); + + it("restores an overwritten env var after invalid snapshot validation", async () => { + await withTempHome(async (home) => { + await writeOpenClawConfig(home, { + env: { vars: { PRE_EXISTING: "new-value" } }, + gateway: { port: "invalid" }, + }); + + const env = { + HOME: home, + PRE_EXISTING: "original-value", + } as NodeJS.ProcessEnv; + const io = createConfigIO({ + env, + homedir: () => home, + logger: { warn: () => {}, error: () => {} }, + }); + + const snapshot = await io.readConfigFileSnapshot(); + + expect(snapshot.valid).toBe(false); + expect(env.PRE_EXISTING).toBe("original-value"); + }); + }); +}); diff --git a/src/config/io.snapshot.ts b/src/config/io.snapshot.ts index 707070d7ad4..dd8eb1630eb 100644 --- a/src/config/io.snapshot.ts +++ b/src/config/io.snapshot.ts @@ -182,6 +182,12 @@ export async function readConfigFileSnapshotInternal( const legacyIssues = await deps.measure("config.snapshot.read.legacy-issues", () => collectInvalidConfigLegacyIssues(effectiveConfigRaw, effectiveParsed), ); + // Invalid snapshots stay inspectable, but rejected env.vars must not become runtime state. + restoreEnvChangesIfUnchanged({ + env: deps.env, + before: envBeforeRead, + after: snapshotEnv(deps.env), + }); return await finalizeReadConfigSnapshotInternalResult(deps, { snapshot: createConfigFileSnapshot({ path: configPath, From fceae4d8aa04d98963d1b928da7863c2c7b3e2f1 Mon Sep 17 00:00:00 2001 From: chengzhichao-xydt Date: Sat, 18 Jul 2026 17:42:19 +0800 Subject: [PATCH 10/76] fix(ui): bound managed outgoing image fetches (#108116) Co-authored-by: Peter Steinberger Co-authored-by: chengzhichao-xydt --- .../chat/components/chat-message.test.ts | 130 ++++++++++++++++++ ui/src/pages/chat/components/chat-message.ts | 45 ++++-- 2 files changed, 161 insertions(+), 14 deletions(-) diff --git a/ui/src/pages/chat/components/chat-message.test.ts b/ui/src/pages/chat/components/chat-message.test.ts index 451f2a39c0d..b8137990f2d 100644 --- a/ui/src/pages/chat/components/chat-message.test.ts +++ b/ui/src/pages/chat/components/chat-message.test.ts @@ -105,6 +105,12 @@ function expectSameOriginGet(init: RequestInit | undefined) { expect(init?.method).toBe("GET"); } +function rejectWhenAborted(signal: AbortSignal, rejection: () => Error): Promise { + return new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(rejection()), { once: true }); + }); +} + function renderAssistantMessage( container: HTMLElement, message: unknown, @@ -2678,6 +2684,130 @@ describe("grouped chat rendering", () => { expectSameOriginGet(fetchInit); }); + it("aborts a stalled managed outgoing image fetch after the deadline", async () => { + vi.useFakeTimers(); + const managedChatImageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { + const signal = init?.signal; + if (!signal) { + throw new Error("missing managed image signal"); + } + return await rejectWhenAborted(signal, () => { + const reason = signal.reason; + return reason instanceof Error ? reason : new Error("managed image fetch aborted"); + }); + }); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + + const container = document.createElement("div"); + renderAssistantMessage( + container, + { + role: "assistant", + content: [ + { + type: "image", + url: managedChatImageUrl, + alt: "Generated image timeout", + width: 1, + height: 1, + }, + ], + timestamp: Date.now(), + }, + { + showToolCalls: false, + assistantAttachmentAuthToken: "test-auth-token", + }, + ); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const [, fetchInit] = requireFetchCallForUrl(fetchMock, managedChatImageUrl); + expect(fetchInit?.signal?.aborted).toBe(false); + expectSameOriginGet(fetchInit); + + await vi.advanceTimersByTimeAsync(29_999); + expect(fetchInit?.signal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(fetchInit?.signal?.aborted).toBe(true); + + await vi.advanceTimersByTimeAsync(0); + expect(container.querySelector(".chat-message-image")).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("falls back when a managed outgoing image body stalls after headers", async () => { + vi.useFakeTimers(); + const managedChatImageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { + const signal = init?.signal; + if (!signal) { + throw new Error("missing managed image signal"); + } + return { + ok: true, + blob: async () => + await rejectWhenAborted( + signal, + () => new DOMException("The operation was aborted.", "AbortError"), + ), + }; + }); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + + const container = document.createElement("div"); + renderAssistantMessage(container, { + role: "assistant", + content: [ + { + type: "image", + url: managedChatImageUrl, + alt: "Generated image body stall", + width: 1, + height: 1, + }, + ], + timestamp: Date.now(), + }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const [, fetchInit] = requireFetchCallForUrl(fetchMock, managedChatImageUrl); + expect(fetchInit?.signal?.aborted).toBe(false); + expectSameOriginGet(fetchInit); + + await vi.advanceTimersByTimeAsync(30_000); + expect(fetchInit?.signal?.aborted).toBe(true); + + await vi.advanceTimersByTimeAsync(0); + expect(container.querySelector(".chat-message-image")).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("treats a failed managed outgoing image fetch as a missing preview", async () => { + const managedChatImageUrl = `/api/chat/media/outgoing/agent%3Amain%3Amain/${crypto.randomUUID()}/full`; + const fetchMock = vi.fn(async () => { + throw new Error("gateway unavailable"); + }); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + + const container = document.createElement("div"); + renderAssistantMessage(container, { + role: "assistant", + content: [ + { + type: "image", + url: managedChatImageUrl, + alt: "Unavailable generated image", + }, + ], + timestamp: Date.now(), + }); + + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1)); + await flushAssistantAttachmentAvailabilityChecks(); + expect(container.querySelector(".chat-message-image")).toBeNull(); + }); + it("bounds managed outgoing image blob URLs with least-recently-used eviction", async () => { const imageUrls = Array.from( { length: 65 }, diff --git a/ui/src/pages/chat/components/chat-message.ts b/ui/src/pages/chat/components/chat-message.ts index 9446cf4eb46..27ce735b6d5 100644 --- a/ui/src/pages/chat/components/chat-message.ts +++ b/ui/src/pages/chat/components/chat-message.ts @@ -267,6 +267,7 @@ const managedImageBlobUrlResolvedCache = new Map(); const managedImageBlobUrlMissCache = new Map(); const MANAGED_IMAGE_BLOB_URL_CACHE_MAX_ENTRIES = 64; const MANAGED_IMAGE_BLOB_URL_MISS_RETRY_MS = 5_000; +const MANAGED_OUTGOING_IMAGE_FETCH_TIMEOUT_MS = 30_000; function readManagedImageBlobUrl(cacheKey: string): string | undefined { const cached = managedImageBlobUrlResolvedCache.get(cacheKey); @@ -1669,23 +1670,39 @@ async function resolveManagedOutgoingImageBlobUrl( if (requesterSessionKey) { headers.set("x-openclaw-requester-session-key", requesterSessionKey); } - const res = await fetch(fetchUrl, { - method: "GET", - headers, - credentials: "same-origin", - }); - if (!res.ok) { - cacheManagedImageBlobUrlMiss(cacheKey); - return null; - } - const blob = await res.blob(); - if (!blob.type.startsWith("image/")) { + const controller = new AbortController(); + const timeout = setTimeout(() => { + controller.abort( + new DOMException("managed outgoing image fetch timed out", "TimeoutError"), + ); + }, MANAGED_OUTGOING_IMAGE_FETCH_TIMEOUT_MS); + try { + const res = await fetch(fetchUrl, { + method: "GET", + headers, + credentials: "same-origin", + signal: controller.signal, + }); + if (!res.ok) { + cacheManagedImageBlobUrlMiss(cacheKey); + return null; + } + const blob = await res.blob(); + if (!blob.type.startsWith("image/")) { + cacheManagedImageBlobUrlMiss(cacheKey); + return null; + } + const blobUrl = URL.createObjectURL(blob); + cacheManagedImageBlobUrl(cacheKey, blobUrl); + return blobUrl; + } catch { + // The render path treats a missing preview as `nothing`; never reject + // its `until` promise for an optional image fetch or body failure. cacheManagedImageBlobUrlMiss(cacheKey); return null; + } finally { + clearTimeout(timeout); } - const blobUrl = URL.createObjectURL(blob); - cacheManagedImageBlobUrl(cacheKey, blobUrl); - return blobUrl; })().finally(() => { managedImageBlobUrlCache.delete(cacheKey); }); From 95d632e644b3c826ca74b76494857f9e9a0fd4e8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 10:44:01 +0100 Subject: [PATCH 11/76] fix(security): bound skill-file audit read via safe reader (restore removed fs dependency) (#110589) --- src/security/audit-extra.async.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/security/audit-extra.async.ts b/src/security/audit-extra.async.ts index dfb66112a2f..6a7c7f4819c 100644 --- a/src/security/audit-extra.async.ts +++ b/src/security/audit-extra.async.ts @@ -3,7 +3,6 @@ * * These functions perform I/O (filesystem, config reads) to detect security issues. */ -import fs from "node:fs/promises"; import path from "node:path"; import { clearTimeout as clearNodeTimeout, setTimeout as setNodeTimeout } from "node:timers"; import { @@ -99,6 +98,10 @@ function expandTilde(p: string, env: NodeJS.ProcessEnv): string | null { } const MAX_PLUGIN_MANIFEST_BYTES = 1024 * 1024; +// Skill file audit reads are bounded like other audit reads; matches the +// workspace loader's DEFAULT_MAX_SKILL_FILE_BYTES so oversized SKILL.md files +// cannot force an unbounded read during the code-safety scan. +const MAX_SKILL_AUDIT_FILE_BYTES = 256_000; async function readPluginManifestExtensions(pluginPath: string): Promise { const manifestPath = path.join(pluginPath, "package.json"); @@ -194,7 +197,10 @@ async function getSkillCodeSafetySummary(params: { dirPath: params.dirPath, summaryCache: params.summaryCache, }), - fs.readFile(params.skillFilePath, "utf-8"), + readRegularFile({ + filePath: params.skillFilePath, + maxBytes: MAX_SKILL_AUDIT_FILE_BYTES, + }).then(({ buffer }) => buffer.toString("utf-8")), loadSkillScannerModule(), ]); const skillFindings = [ From a941d3ab727e3f0d789f51dd6abfb8c433bcdee6 Mon Sep 17 00:00:00 2001 From: tzy-17 Date: Sat, 18 Jul 2026 17:44:31 +0800 Subject: [PATCH 12/76] fix(shared): reject HTTP status codes outside the 100-599 range (#110554) * fix(shared): reject HTTP status codes outside the 100-599 range extractHttpStatusMatch only checked Number.isFinite, so 3-digit sequences such as "000", "099", "600", and "999" were accepted as valid HTTP statuses. Downstream code then surfaced them as "HTTP 0" in user-visible messages and fed them into retry/classification branches (e.g. isCloudflareOrHtmlErrorPage checks `code < 500` and `code < 600`). Restrict the extracted code to the IANA HTTP status code range (100-599). Co-authored-by: Patrick Erichsen * fix(shared): validate all parsed HTTP status prefixes --------- Co-authored-by: Patrick Erichsen Co-authored-by: Peter Steinberger --- src/shared/assistant-error-format.test.ts | 87 +++++++++++++++++++++++ src/shared/assistant-error-format.ts | 18 +++-- 2 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 src/shared/assistant-error-format.test.ts diff --git a/src/shared/assistant-error-format.test.ts b/src/shared/assistant-error-format.test.ts new file mode 100644 index 00000000000..85d8f76a2e1 --- /dev/null +++ b/src/shared/assistant-error-format.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { + extractLeadingHttpStatus, + extractProviderWrappedHttpStatus, + formatRawAssistantErrorForUi, + parseApiErrorInfo, +} from "./assistant-error-format.js"; + +describe("extractLeadingHttpStatus", () => { + it("accepts status codes in the valid HTTP range 100-599", () => { + expect(extractLeadingHttpStatus("100 everything is fine")).toEqual({ + code: 100, + rest: "everything is fine", + }); + expect(extractLeadingHttpStatus("500 internal error")).toEqual({ + code: 500, + rest: "internal error", + }); + expect(extractLeadingHttpStatus("599 something rest")).toEqual({ + code: 599, + rest: "something rest", + }); + }); + + it("rejects 3-digit sequences outside the HTTP status code range", () => { + // 000 / 099 / 999 / 600 — the regex would capture these as 3-digit + // numbers, but they are not valid HTTP statuses and should not be + // surfaced as "HTTP " in user-visible messages or in retry + // classification. + expect(extractLeadingHttpStatus("000 something")).toBeNull(); + expect(extractLeadingHttpStatus("099 something")).toBeNull(); + expect(extractLeadingHttpStatus("600 something")).toBeNull(); + expect(extractLeadingHttpStatus("999 something")).toBeNull(); + }); + + it("rejects strings that do not start with a 3-digit HTTP status", () => { + expect(extractLeadingHttpStatus("no status here")).toBeNull(); + expect(extractLeadingHttpStatus("")).toBeNull(); + }); +}); + +describe("extractProviderWrappedHttpStatus", () => { + it("accepts provider-wrapped statuses inside the valid HTTP range", () => { + expect(extractProviderWrappedHttpStatus("OpenAI API error (503): service down")).toEqual({ + code: 503, + rest: "service down", + }); + expect(extractProviderWrappedHttpStatus("API error (429): rate limited")).toEqual({ + code: 429, + rest: "rate limited", + }); + }); + + it("rejects provider-wrapped statuses outside the valid HTTP range", () => { + expect(extractProviderWrappedHttpStatus("API error (000): something")).toBeNull(); + expect(extractProviderWrappedHttpStatus("API error (999): something")).toBeNull(); + expect(extractProviderWrappedHttpStatus("API error (600): something")).toBeNull(); + }); +}); + +describe("HTTP status consumers", () => { + it("formats only status lines inside the HTTP range", () => { + expect(formatRawAssistantErrorForUi("100 Continue")).toBe("HTTP 100: Continue"); + expect(formatRawAssistantErrorForUi("599 Provider Error")).toBe("HTTP 599: Provider Error"); + expect(formatRawAssistantErrorForUi("000 Invalid")).toBe("000 Invalid"); + expect(formatRawAssistantErrorForUi("600 Invalid")).toBe("600 Invalid"); + expect(formatRawAssistantErrorForUi("999 Invalid")).toBe("999 Invalid"); + }); + + it("does not attach invalid status prefixes to API payloads", () => { + const payload = '{"type":"error","error":{"type":"server_error","message":"Provider failed."}}'; + + expect(parseApiErrorInfo(`599 ${payload}`)).toMatchObject({ + httpCode: "599", + type: "server_error", + message: "Provider failed.", + }); + expect(formatRawAssistantErrorForUi(`599 ${payload}`)).toBe( + "HTTP 599 server_error: Provider failed.", + ); + + for (const code of ["000", "600", "999"]) { + expect(parseApiErrorInfo(`${code} ${payload}`)).toBeNull(); + expect(formatRawAssistantErrorForUi(`${code} ${payload}`)).toBe(`${code} ${payload}`); + } + }); +}); diff --git a/src/shared/assistant-error-format.ts b/src/shared/assistant-error-format.ts index 46a30479a41..5ed7532a862 100644 --- a/src/shared/assistant-error-format.ts +++ b/src/shared/assistant-error-format.ts @@ -1,4 +1,3 @@ -import { expectDefined } from "@openclaw/normalization-core"; // Assistant error formatting helpers normalize assistant-visible error payloads. import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; const ERROR_PAYLOAD_PREFIX_RE = @@ -107,7 +106,7 @@ function extractHttpStatusMatch( return null; } const code = Number(match[1]); - if (!Number.isFinite(code)) { + if (!Number.isInteger(code) || code < 100 || code > 599) { return null; } return { code, rest: (match[2] ?? "").trim() }; @@ -174,10 +173,10 @@ export function parseApiErrorInfo(raw?: string): ApiErrorInfo | null { let httpCode: string | undefined; let candidate = trimmed; - const httpPrefixMatch = candidate.match(/^(\d{3})\s+(.+)$/s); - if (httpPrefixMatch) { - httpCode = httpPrefixMatch[1]; - candidate = expectDefined(httpPrefixMatch[2], "http prefix match capture group 2").trim(); + const httpPrefix = extractHttpStatusMatch(candidate.match(/^(\d{3})\s+(.+)$/s)); + if (httpPrefix) { + httpCode = String(httpPrefix.code); + candidate = httpPrefix.rest; } const payload = parseApiErrorPayload(candidate); @@ -249,11 +248,10 @@ export function formatRawAssistantErrorForUi(raw?: string): string { ); } - const httpMatch = trimmed.match(HTTP_STATUS_PREFIX_RE); + const httpMatch = extractHttpStatusMatch(trimmed.match(HTTP_STATUS_PREFIX_RE)); if (httpMatch) { - const rest = expectDefined(httpMatch[2], "http match capture group 2").trim(); - if (!rest.startsWith("{")) { - return `HTTP ${httpMatch[1]}: ${rest}`; + if (!httpMatch.rest.startsWith("{")) { + return `HTTP ${httpMatch.code}: ${httpMatch.rest}`; } } From 7752e33f799e7d207aca195a2f1ad835541c5958 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 02:40:37 -0700 Subject: [PATCH 13/76] Revert "fix(security): restore fs import lost in bounded manifest-read merge" This reverts commit 39528edd745a484457e2add67155f611b2f7da5b. That push was an accident: the fs-import fix it described had already landed on main, and a rebase reduced the commit to unrelated in-progress gateway-protocol schema edits (nodes/session-placement) that were staged in the pushing worktree and belong to another work stream. Restore the reviewed main state; the protocol work can land through its own PR. --- .../gateway-protocol/src/schema/nodes.test.ts | 34 +++++ packages/gateway-protocol/src/schema/nodes.ts | 17 +-- .../src/schema/session-placement.test.ts | 24 ++++ .../src/schema/session-placement.ts | 128 +++++++----------- 4 files changed, 116 insertions(+), 87 deletions(-) create mode 100644 packages/gateway-protocol/src/schema/nodes.test.ts diff --git a/packages/gateway-protocol/src/schema/nodes.test.ts b/packages/gateway-protocol/src/schema/nodes.test.ts new file mode 100644 index 00000000000..5735413cc91 --- /dev/null +++ b/packages/gateway-protocol/src/schema/nodes.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { validateNodeInvokeProgressParams } from "../index.js"; + +describe("node protocol schemas", () => { + it("accepts bounded progress chunks and rejects extra fields", () => { + expect( + validateNodeInvokeProgressParams({ + invokeId: "invoke-1", + nodeId: "node-1", + seq: 0, + chunk: "stdout line", + }), + ).toBe(true); + + expect( + validateNodeInvokeProgressParams({ + invokeId: "invoke-1", + nodeId: "node-1", + seq: 0, + chunk: "x".repeat(16 * 1024 + 1), + }), + ).toBe(false); + + expect( + validateNodeInvokeProgressParams({ + invokeId: "invoke-1", + nodeId: "node-1", + seq: 0, + chunk: "stdout line", + extra: "not allowed", + }), + ).toBe(false); + }); +}); diff --git a/packages/gateway-protocol/src/schema/nodes.ts b/packages/gateway-protocol/src/schema/nodes.ts index 40e8ce9fea5..47fd1a300ec 100644 --- a/packages/gateway-protocol/src/schema/nodes.ts +++ b/packages/gateway-protocol/src/schema/nodes.ts @@ -164,16 +164,13 @@ export const NodeInvokeResultParamsSchema = closedObject({ }); /** Ordered UTF-8 output emitted while a node command invocation is running. */ -export const NodeInvokeProgressParamsSchema = Type.Object( - { - invokeId: NonEmptyString, - nodeId: NonEmptyString, - seq: Type.Integer({ minimum: 0 }), - // Empty chunks are liveness heartbeats for captured stderr or capped stdout. - chunk: Type.String({ maxLength: 16 * 1024 }), - }, - { additionalProperties: false }, -); +export const NodeInvokeProgressParamsSchema = closedObject({ + invokeId: NonEmptyString, + nodeId: NonEmptyString, + seq: Type.Integer({ minimum: 0 }), + // Empty chunks are liveness heartbeats for captured stderr or capped stdout. + chunk: Type.String({ maxLength: 16 * 1024 }), +}); /** Generic node event envelope accepted by the gateway. */ export const NodeEventParamsSchema = closedObject({ diff --git a/packages/gateway-protocol/src/schema/session-placement.test.ts b/packages/gateway-protocol/src/schema/session-placement.test.ts index 8943d06e094..789b72bb629 100644 --- a/packages/gateway-protocol/src/schema/session-placement.test.ts +++ b/packages/gateway-protocol/src/schema/session-placement.test.ts @@ -282,4 +282,28 @@ describe("session dispatch protocol schemas", () => { }), ).toBe(false); }); + + it("rejects extra fields in dispatch params and results", () => { + const active = { + state: "active" as const, + ...basePlacement, + ...workerOwnedFields, + }; + expect( + validateSessionsDispatchResult({ + ok: true, + key: "agent:main:dispatch", + sessionId: "session-1", + placement: active, + extra: true, + }), + ).toBe(false); + expect( + validateSessionsDispatchParams({ + key: "agent:main:dispatch", + profileId: "development", + extra: true, + }), + ).toBe(false); + }); }); diff --git a/packages/gateway-protocol/src/schema/session-placement.ts b/packages/gateway-protocol/src/schema/session-placement.ts index 65ff59e7989..34acbab4ff7 100644 --- a/packages/gateway-protocol/src/schema/session-placement.ts +++ b/packages/gateway-protocol/src/schema/session-placement.ts @@ -1,5 +1,6 @@ import type { Static } from "typebox"; import { Type } from "typebox"; +import { closedObject } from "./closed-object.js"; import { NonEmptyString } from "./primitives.js"; import { SESSION_PLACEMENT_STATES } from "./session-placement-state.js"; @@ -77,85 +78,64 @@ const TerminalSessionPlacementProperties = { function createUnownedSessionPlacementSchema( state: State, ) { - return Type.Object( - { state: Type.Literal(state), ...SessionPlacementTimingProperties }, - { additionalProperties: false }, - ); + return closedObject({ state: Type.Literal(state), ...SessionPlacementTimingProperties }); } function createWorkerOwnedSessionPlacementSchema< const State extends "active" | "draining" | "reconciling", >(state: State) { - return Type.Object( - { - state: Type.Literal(state), - ...SessionPlacementTimingProperties, - environmentId: NonEmptyString, - activeOwnerEpoch: SessionPlacementOwnerEpochSchema, - workerBundleHash: WorkerBundleHashSchema, - ...SessionPlacementWorkspaceProperties, - ...SessionPlacementAckProperties, - }, - { additionalProperties: false }, - ); + return closedObject({ + state: Type.Literal(state), + ...SessionPlacementTimingProperties, + environmentId: NonEmptyString, + activeOwnerEpoch: SessionPlacementOwnerEpochSchema, + workerBundleHash: WorkerBundleHashSchema, + ...SessionPlacementWorkspaceProperties, + ...SessionPlacementAckProperties, + }); } export const LocalSessionPlacementSchema = createUnownedSessionPlacementSchema("local"); export const RequestedSessionPlacementSchema = createUnownedSessionPlacementSchema("requested"); -export const ProvisioningSessionPlacementSchema = Type.Object( - { - state: Type.Literal("provisioning"), - ...SessionPlacementTimingProperties, - environmentId: Type.Optional(NonEmptyString), - }, - { additionalProperties: false }, -); +export const ProvisioningSessionPlacementSchema = closedObject({ + state: Type.Literal("provisioning"), + ...SessionPlacementTimingProperties, + environmentId: Type.Optional(NonEmptyString), +}); -export const SyncingSessionPlacementSchema = Type.Object( - { - state: Type.Literal("syncing"), - ...SessionPlacementTimingProperties, - environmentId: NonEmptyString, - workerBundleHash: WorkerBundleHashSchema, - }, - { additionalProperties: false }, -); +export const SyncingSessionPlacementSchema = closedObject({ + state: Type.Literal("syncing"), + ...SessionPlacementTimingProperties, + environmentId: NonEmptyString, + workerBundleHash: WorkerBundleHashSchema, +}); -export const StartingSessionPlacementSchema = Type.Object( - { - state: Type.Literal("starting"), - ...SessionPlacementTimingProperties, - environmentId: NonEmptyString, - workerBundleHash: WorkerBundleHashSchema, - ...SessionPlacementWorkspaceProperties, - }, - { additionalProperties: false }, -); +export const StartingSessionPlacementSchema = closedObject({ + state: Type.Literal("starting"), + ...SessionPlacementTimingProperties, + environmentId: NonEmptyString, + workerBundleHash: WorkerBundleHashSchema, + ...SessionPlacementWorkspaceProperties, +}); export const ActiveWorkerSessionPlacementSchema = createWorkerOwnedSessionPlacementSchema("active"); export const DrainingSessionPlacementSchema = createWorkerOwnedSessionPlacementSchema("draining"); export const ReconcilingSessionPlacementSchema = createWorkerOwnedSessionPlacementSchema("reconciling"); -export const ReclaimedSessionPlacementSchema = Type.Object( - { - state: Type.Literal("reclaimed"), - ...SessionPlacementTimingProperties, - ...TerminalSessionPlacementProperties, - }, - { additionalProperties: false }, -); +export const ReclaimedSessionPlacementSchema = closedObject({ + state: Type.Literal("reclaimed"), + ...SessionPlacementTimingProperties, + ...TerminalSessionPlacementProperties, +}); -export const FailedSessionPlacementSchema = Type.Object( - { - state: Type.Literal("failed"), - ...SessionPlacementTimingProperties, - ...TerminalSessionPlacementProperties, - recoveryError: NonEmptyString, - }, - { additionalProperties: false }, -); +export const FailedSessionPlacementSchema = closedObject({ + state: Type.Literal("failed"), + ...SessionPlacementTimingProperties, + ...TerminalSessionPlacementProperties, + recoveryError: NonEmptyString, +}); /** Gateway-visible placement projection; `state` remains the closed discriminator. */ export const SessionPlacementSchema = Type.Union([ @@ -172,25 +152,19 @@ export const SessionPlacementSchema = Type.Union([ ]); /** Requests one-way dispatch of an existing local session to a configured worker profile. */ -export const SessionsDispatchParamsSchema = Type.Object( - { - key: NonEmptyString, - agentId: Type.Optional(NonEmptyString), - profileId: NonEmptyString, - }, - { additionalProperties: false }, -); +export const SessionsDispatchParamsSchema = closedObject({ + key: NonEmptyString, + agentId: Type.Optional(NonEmptyString), + profileId: NonEmptyString, +}); /** Result returned once session dispatch reaches durable worker ownership. */ -export const SessionsDispatchResultSchema = Type.Object( - { - ok: Type.Literal(true), - key: NonEmptyString, - sessionId: NonEmptyString, - placement: ActiveWorkerSessionPlacementSchema, - }, - { additionalProperties: false }, -); +export const SessionsDispatchResultSchema = closedObject({ + ok: Type.Literal(true), + key: NonEmptyString, + sessionId: NonEmptyString, + placement: ActiveWorkerSessionPlacementSchema, +}); /** Requests safe workspace reconciliation and teardown of an active cloud worker. */ export const SessionsReclaimParamsSchema = Type.Object( From fcdf06ed29449355fa636566a30b27416327eaab Mon Sep 17 00:00:00 2001 From: zengLingbiao Date: Sat, 18 Jul 2026 17:49:53 +0800 Subject: [PATCH 14/76] fix(config): ignore inherited keys when restoring env refs (#109463) * fix(config): use Object.hasOwn in restoreEnvVarRefs to avoid prototype pollution When key comes from Object.entries(incoming) on user config data, 'key in parsed' traverses the prototype chain. If a config key collides with an Object.prototype property (e.g. 'toString'), the check incorrectly returns true and calls restoreEnvVarRefs with a prototype function instead of a config value. Replace with Object.hasOwn(parsed, key) to check only own properties. * test(config): prove inherited env refs are ignored --------- Co-authored-by: Peter Steinberger Co-authored-by: Peter Steinberger --- src/config/env-preserve.test.ts | 12 ++++++++++++ src/config/env-preserve.ts | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/config/env-preserve.test.ts b/src/config/env-preserve.test.ts index 3053cb2de6c..a41f1273f5d 100644 --- a/src/config/env-preserve.test.ts +++ b/src/config/env-preserve.test.ts @@ -833,4 +833,16 @@ describe("restoreEnvVarRefs", () => { resolved: "${MY_TOKEN}", // should restore ref }); }); + + it("does not restore env refs from inherited parsed properties", () => { + // isPlainObject accepts custom prototypes, but only own properties represent authored config. + const parsed = Object.create({ + toString: "${TEST_VALUE}", + }) as Record; + const testEnv = { TEST_VALUE: "resolved-value" } as unknown as NodeJS.ProcessEnv; + + expect(restoreEnvVarRefs({ toString: "resolved-value" }, parsed, testEnv)).toEqual({ + toString: "resolved-value", + }); + }); }); diff --git a/src/config/env-preserve.ts b/src/config/env-preserve.ts index b56a514db9f..fae54c623f3 100644 --- a/src/config/env-preserve.ts +++ b/src/config/env-preserve.ts @@ -820,7 +820,7 @@ export function restoreEnvVarRefs( if (isPlainObject(incoming) && isPlainObject(parsed)) { const result: Record = {}; for (const [key, value] of Object.entries(incoming)) { - if (key in parsed) { + if (Object.hasOwn(parsed, key)) { result[key] = restoreEnvVarRefs(value, parsed[key], env); } else { // New key added by caller — keep as-is From a068d38b663e28a8c5f0731699c935ba9c0ad14c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 10:56:37 +0100 Subject: [PATCH 15/76] refactor(plugins)!: remove experimental workspaces plugin (#110416) * refactor(plugins)!: remove experimental workspaces plugin * fix(doctor): remove retired workspaces plugin state * refactor(doctor): settle retired-workspaces cleanup before legacy-state detect * chore(i18n): leave workflow-owned locale artifacts to the locale refresh * ci: retrigger after GitHub scheduling outage --- .github/labeler.yml | 5 - config/max-lines-baseline.txt | 3 - docs/docs.json | 1 - docs/docs_map.md | 20 - docs/plugins/plugin-inventory.md | 4 +- docs/plugins/reference.md | 2 +- docs/plugins/reference/workspaces.md | 19 - docs/web/workspaces.md | 132 -- extensions/workspaces/index.ts | 81 -- extensions/workspaces/openclaw.plugin.json | 35 - extensions/workspaces/package.json | 27 - .../workspaces/src/asset-tokens.test.ts | 23 - extensions/workspaces/src/asset-tokens.ts | 61 - extensions/workspaces/src/binding-contract.ts | 71 -- extensions/workspaces/src/broadcast.ts | 27 - .../workspaces/src/cli-registration.test.ts | 35 - extensions/workspaces/src/cli.test.ts | 427 ------- extensions/workspaces/src/cli.ts | 509 -------- extensions/workspaces/src/data-read.test.ts | 162 --- extensions/workspaces/src/data-read.ts | 167 --- .../workspaces/src/default-workspace.test.ts | 61 - .../workspaces/src/default-workspace.ts | 108 -- extensions/workspaces/src/gateway.test.ts | 297 ----- extensions/workspaces/src/gateway.ts | 838 ------------- extensions/workspaces/src/http-route.ts | 32 - extensions/workspaces/src/manifest.test.ts | 215 ---- extensions/workspaces/src/manifest.ts | 369 ------ extensions/workspaces/src/scaffold.test.ts | 32 - extensions/workspaces/src/scaffold.ts | 177 --- extensions/workspaces/src/schema.test.ts | 164 --- extensions/workspaces/src/schema.ts | 476 -------- extensions/workspaces/src/serve.test.ts | 435 ------- extensions/workspaces/src/serve.ts | 296 ----- extensions/workspaces/src/store.test.ts | 339 ------ extensions/workspaces/src/store.ts | 313 ----- extensions/workspaces/src/tools.test.ts | 422 ------- extensions/workspaces/src/tools.ts | 919 -------------- .../src/ui-gateway-integration.test.ts | 251 ---- pnpm-lock.yaml | 13 - src/flows/doctor-core-checks.ts | 4 +- src/flows/doctor-health-contributions.test.ts | 8 +- src/flows/doctor-health-contributions.ts | 5 +- ...tor-removed-workspaces-state-check.test.ts | 85 ++ .../doctor-removed-workspaces-state-check.ts | 72 ++ src/plugins/bundled-plugin-metadata.test.ts | 1 - .../workspace-custom-widget.test.ts | 339 ------ ui/src/components/workspace-custom-widget.ts | 360 ------ .../components/workspace-widget-cell.test.ts | 173 --- ui/src/components/workspace-widget-cell.ts | 393 ------ ui/src/e2e/workspace-chart.e2e.test.ts | 131 -- .../e2e/workspace-custom-widget.e2e.test.ts | 495 -------- ui/src/e2e/workspace-full-bleed.e2e.test.ts | 135 --- ui/src/e2e/workspace-operations.e2e.test.ts | 193 --- ui/src/e2e/workspace-preview.e2e.test.ts | 147 --- ui/src/i18n/locales/en.ts | 127 -- ui/src/lib/workspace/approval-state.ts | 13 - ui/src/lib/workspace/bridge.test.ts | 363 ------ ui/src/lib/workspace/bridge.ts | 326 ----- ui/src/lib/workspace/grid.test.ts | 91 -- ui/src/lib/workspace/grid.ts | 230 ---- ui/src/lib/workspace/index.test.ts | 872 ------------- ui/src/lib/workspace/index.ts | 838 ------------- ui/src/lib/workspace/tab-selection.ts | 73 -- ui/src/lib/workspace/types.ts | 118 -- ui/src/lib/workspace/widgets/activity.ts | 93 -- ui/src/lib/workspace/widgets/agent-status.ts | 122 -- ui/src/lib/workspace/widgets/chart.ts | 275 ----- ui/src/lib/workspace/widgets/cron.ts | 93 -- .../widgets/custom-widget-approvals.ts | 120 -- ui/src/lib/workspace/widgets/iframe-embed.ts | 120 -- ui/src/lib/workspace/widgets/index.ts | 43 - ui/src/lib/workspace/widgets/instances.ts | 101 -- ui/src/lib/workspace/widgets/markdown.ts | 36 - ui/src/lib/workspace/widgets/preview.ts | 105 -- ui/src/lib/workspace/widgets/sessions.ts | 94 -- ui/src/lib/workspace/widgets/stat-card.ts | 80 -- ui/src/lib/workspace/widgets/table.ts | 103 -- ui/src/lib/workspace/widgets/types.ts | 73 -- ui/src/lib/workspace/widgets/usage.ts | 38 - ui/src/lib/workspace/widgets/widgets.test.ts | 585 --------- ui/src/pages/plugin/plugin-page.test.ts | 35 +- ui/src/pages/plugin/plugin-page.ts | 9 - .../pages/plugin/workspace-controller.test.ts | 23 - ui/src/pages/plugin/workspace-controller.ts | 15 - ui/src/pages/plugin/workspace-view.test.ts | 519 -------- ui/src/pages/plugin/workspace-view.ts | 1080 ----------------- ui/src/styles/workspace.css | 1043 ---------------- 87 files changed, 188 insertions(+), 17777 deletions(-) delete mode 100644 docs/plugins/reference/workspaces.md delete mode 100644 docs/web/workspaces.md delete mode 100644 extensions/workspaces/index.ts delete mode 100644 extensions/workspaces/openclaw.plugin.json delete mode 100644 extensions/workspaces/package.json delete mode 100644 extensions/workspaces/src/asset-tokens.test.ts delete mode 100644 extensions/workspaces/src/asset-tokens.ts delete mode 100644 extensions/workspaces/src/binding-contract.ts delete mode 100644 extensions/workspaces/src/broadcast.ts delete mode 100644 extensions/workspaces/src/cli-registration.test.ts delete mode 100644 extensions/workspaces/src/cli.test.ts delete mode 100644 extensions/workspaces/src/cli.ts delete mode 100644 extensions/workspaces/src/data-read.test.ts delete mode 100644 extensions/workspaces/src/data-read.ts delete mode 100644 extensions/workspaces/src/default-workspace.test.ts delete mode 100644 extensions/workspaces/src/default-workspace.ts delete mode 100644 extensions/workspaces/src/gateway.test.ts delete mode 100644 extensions/workspaces/src/gateway.ts delete mode 100644 extensions/workspaces/src/http-route.ts delete mode 100644 extensions/workspaces/src/manifest.test.ts delete mode 100644 extensions/workspaces/src/manifest.ts delete mode 100644 extensions/workspaces/src/scaffold.test.ts delete mode 100644 extensions/workspaces/src/scaffold.ts delete mode 100644 extensions/workspaces/src/schema.test.ts delete mode 100644 extensions/workspaces/src/schema.ts delete mode 100644 extensions/workspaces/src/serve.test.ts delete mode 100644 extensions/workspaces/src/serve.ts delete mode 100644 extensions/workspaces/src/store.test.ts delete mode 100644 extensions/workspaces/src/store.ts delete mode 100644 extensions/workspaces/src/tools.test.ts delete mode 100644 extensions/workspaces/src/tools.ts delete mode 100644 extensions/workspaces/src/ui-gateway-integration.test.ts create mode 100644 src/flows/doctor-removed-workspaces-state-check.test.ts create mode 100644 src/flows/doctor-removed-workspaces-state-check.ts delete mode 100644 ui/src/components/workspace-custom-widget.test.ts delete mode 100644 ui/src/components/workspace-custom-widget.ts delete mode 100644 ui/src/components/workspace-widget-cell.test.ts delete mode 100644 ui/src/components/workspace-widget-cell.ts delete mode 100644 ui/src/e2e/workspace-chart.e2e.test.ts delete mode 100644 ui/src/e2e/workspace-custom-widget.e2e.test.ts delete mode 100644 ui/src/e2e/workspace-full-bleed.e2e.test.ts delete mode 100644 ui/src/e2e/workspace-operations.e2e.test.ts delete mode 100644 ui/src/e2e/workspace-preview.e2e.test.ts delete mode 100644 ui/src/lib/workspace/approval-state.ts delete mode 100644 ui/src/lib/workspace/bridge.test.ts delete mode 100644 ui/src/lib/workspace/bridge.ts delete mode 100644 ui/src/lib/workspace/grid.test.ts delete mode 100644 ui/src/lib/workspace/grid.ts delete mode 100644 ui/src/lib/workspace/index.test.ts delete mode 100644 ui/src/lib/workspace/index.ts delete mode 100644 ui/src/lib/workspace/tab-selection.ts delete mode 100644 ui/src/lib/workspace/types.ts delete mode 100644 ui/src/lib/workspace/widgets/activity.ts delete mode 100644 ui/src/lib/workspace/widgets/agent-status.ts delete mode 100644 ui/src/lib/workspace/widgets/chart.ts delete mode 100644 ui/src/lib/workspace/widgets/cron.ts delete mode 100644 ui/src/lib/workspace/widgets/custom-widget-approvals.ts delete mode 100644 ui/src/lib/workspace/widgets/iframe-embed.ts delete mode 100644 ui/src/lib/workspace/widgets/index.ts delete mode 100644 ui/src/lib/workspace/widgets/instances.ts delete mode 100644 ui/src/lib/workspace/widgets/markdown.ts delete mode 100644 ui/src/lib/workspace/widgets/preview.ts delete mode 100644 ui/src/lib/workspace/widgets/sessions.ts delete mode 100644 ui/src/lib/workspace/widgets/stat-card.ts delete mode 100644 ui/src/lib/workspace/widgets/table.ts delete mode 100644 ui/src/lib/workspace/widgets/types.ts delete mode 100644 ui/src/lib/workspace/widgets/usage.ts delete mode 100644 ui/src/lib/workspace/widgets/widgets.test.ts delete mode 100644 ui/src/pages/plugin/workspace-controller.test.ts delete mode 100644 ui/src/pages/plugin/workspace-controller.ts delete mode 100644 ui/src/pages/plugin/workspace-view.test.ts delete mode 100644 ui/src/pages/plugin/workspace-view.ts delete mode 100644 ui/src/styles/workspace.css diff --git a/.github/labeler.yml b/.github/labeler.yml index a448dc25c9b..b926f4827f9 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -70,11 +70,6 @@ - "extensions/workboard/**" - "docs/plugins/workboard.md" - "docs/plugins/reference/workboard.md" -"plugin: workspaces": - - changed-files: - - any-glob-to-any-file: - - "extensions/workspaces/**" - - "docs/web/workspaces.md" "plugin: migrate-hermes": - changed-files: - any-glob-to-any-file: diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 3364cfb8bf7..433a2aa64af 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -339,8 +339,6 @@ extensions/workboard/src/store-core.ts extensions/workboard/src/store-normalizers.ts extensions/workboard/src/store.test.ts extensions/workboard/src/tools.ts -extensions/workspaces/src/gateway.ts -extensions/workspaces/src/tools.ts extensions/xai/realtime-voice-provider.test.ts extensions/xai/web-search.test.ts extensions/zalo/src/monitor.ts @@ -1155,7 +1153,6 @@ ui/src/pages/config/view.browser.test.ts ui/src/pages/config/view.ts ui/src/pages/cron/view.ts ui/src/pages/new-session/new-session-page.ts -ui/src/pages/plugin/workspace-view.ts ui/src/pages/plugins/plugins-page.ts ui/src/pages/plugins/view.ts ui/src/pages/sessions/sessions-page.ts diff --git a/docs/docs.json b/docs/docs.json index c27d8c5de8e..4497d692185 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1750,7 +1750,6 @@ "pages": [ "web/index", "web/control-ui", - "web/workspaces", "web/dashboard", "web/webchat", "web/tui", diff --git a/docs/docs_map.md b/docs/docs_map.md index d2aa5612e7b..4fd98d4a6a2 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -7263,14 +7263,6 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Surface - H2: Related docs -## plugins/reference/workspaces.md - -- Route: /plugins/reference/workspaces -- Headings: - - H1: Workspaces plugin - - H2: Distribution - - H2: Surface - ## plugins/reference/xai.md - Route: /plugins/reference/xai @@ -10612,15 +10604,3 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Remote use - H2: Configuration reference (WebChat) - H2: Related - -## web/workspaces.md - -- Route: /web/workspaces -- Headings: - - H2: Enable Workspaces - - H2: The default workspace - - H2: Built-in widgets - - H2: Provenance - - H2: Custom widgets - - H2: CLI - - H2: Storage diff --git a/docs/plugins/plugin-inventory.md b/docs/plugins/plugin-inventory.md index ebdc06d7e49..0f889677d34 100644 --- a/docs/plugins/plugin-inventory.md +++ b/docs/plugins/plugin-inventory.md @@ -51,7 +51,7 @@ Each entry lists the package, distribution route, and description. ## Core npm package -69 plugins +68 plugins - **[admin-http-rpc](/plugins/reference/admin-http-rpc)** (`@openclaw/admin-http-rpc`) - included in OpenClaw. OpenClaw admin HTTP RPC endpoint. @@ -185,8 +185,6 @@ Each entry lists the package, distribution route, and description. - **[workboard](/plugins/reference/workboard)** (`@openclaw/workboard`) - included in OpenClaw. Dashboard workboard for agent-owned issues and sessions. -- **[workspaces](/plugins/reference/workspaces)** (`@openclaw/workspaces-plugin`) - included in OpenClaw. Agent-composable Workspaces document and control-plane backend. - - **[xai](/plugins/reference/xai)** (`@openclaw/xai-plugin`) - included in OpenClaw. Adds xAI model provider support to OpenClaw. - **[xiaomi](/plugins/reference/xiaomi)** (`@openclaw/xiaomi-provider`) - included in OpenClaw. Adds Xiaomi, Xiaomi Token Plan model provider support to OpenClaw. diff --git a/docs/plugins/reference.md b/docs/plugins/reference.md index 11258cad202..4d4ccfadaf3 100644 --- a/docs/plugins/reference.md +++ b/docs/plugins/reference.md @@ -15,5 +15,5 @@ This page is generated from `extensions/*/package.json` and pnpm plugins:inventory:gen ``` -Use [Plugin inventory](/plugins/plugin-inventory) to browse all 142 +Use [Plugin inventory](/plugins/plugin-inventory) to browse all 141 generated plugin reference pages by distribution, package, and description. diff --git a/docs/plugins/reference/workspaces.md b/docs/plugins/reference/workspaces.md deleted file mode 100644 index 0014e2083f0..00000000000 --- a/docs/plugins/reference/workspaces.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -summary: "Agent-composable Workspaces document and control-plane backend." -read_when: - - You are installing, configuring, or auditing the workspaces plugin -title: "Workspaces plugin" ---- - -# Workspaces plugin - -Agent-composable Workspaces document and control-plane backend. - -## Distribution - -- Package: `@openclaw/workspaces-plugin` -- Install route: included in OpenClaw - -## Surface - -contracts: `tools` diff --git a/docs/web/workspaces.md b/docs/web/workspaces.md deleted file mode 100644 index 1cfcaa37295..00000000000 --- a/docs/web/workspaces.md +++ /dev/null @@ -1,132 +0,0 @@ ---- -summary: "Agent-composable Workspaces in the Control UI" -read_when: - - Building or rearranging workspace tabs and widgets - - Letting an agent compose a workspace - - Reviewing the custom-widget approval and sandbox model -title: "Workspaces" ---- - -The **Workspaces** tab in the [Control UI](/web/control-ui) is a surface you and your -agents arrange together. Tabs, widgets, their positions on a 12-column grid, and their -data bindings all live in one document. Anything that can edit that document can compose -the workspace: you, the `openclaw workspaces` CLI, or an agent calling `workspace_*` tools. - -Every write goes through the same validated path, so a human's layout and an agent's -layout cannot diverge. Each accepted write bumps a version and broadcasts -`plugin.workspaces.changed`, so an agent's edit appears in an already-open browser without -a reload. - -## Enable Workspaces - -The bundled Workspaces plugin is disabled by default. In the Control UI, open **Plugins**, -find **Workspaces**, and select **Enable**. You can also enable it from the CLI: - -```sh -openclaw plugins enable workspaces -``` - -Enabling the plugin adds the **Workspaces** tab and makes the `openclaw workspaces` CLI -and `workspace_*` agent tools available. Disabling it removes those surfaces without -deleting the workspace database or widget assets. - -## The default workspace - -On first load you get an **Overview** workspace: cost and token cards, instance health, -sessions, cron status, and an activity feed. It is ordinary workspace content — drag it, -collapse it, hide it, or delete it. - -## Built-in widgets - -Thirteen trusted widgets ship with the plugin and render as first-party UI: - -`stat-card`, `markdown`, `table`, `iframe-embed`, `sessions`, `usage`, `cron`, -`instances`, `activity`, `chart`, `preview`, `agent-status`, -`custom-widget-approvals`. - -The `chart` widget renders dependency-free inline SVG as `line`, `bar`, `area`, -`sparkline`, or `gauge`. Bind `value` to a numeric array or to an object shaped like -`{ "points": [1, { "y": 2 }, { "value": 3 }] }`. Set `props.type` to select the -visual and optionally set finite numeric `props.min` and `props.max` bounds. Invalid -types, bounds, points, and series longer than 500 entries render a safe error state; -empty series render an empty state. - -The `preview` widget embeds a live page with reload plus desktop, tablet, and mobile -viewport controls. Set `props.url`, or bind `value` to a URL when the preview target is -data-driven; a binding takes precedence over the prop. Relative and same-origin HTTP(S) -URLs are allowed. External HTTP(S) URLs follow the gateway's external-embed policy, and -other schemes are blocked. Preview frames share the `iframe-embed` sandbox ceiling and -never receive same-origin access. - -The `agent-status` widget presents the current Busy or Idle state from a -`sessions.list` RPC binding. Session-list events refresh it immediately, while the normal -visibility-gated binding poll remains a fallback. The `custom-widget-approvals` widget -lists only pending entries from the Workspaces custom-widget registry. Its Approve and -Reject controls are disabled unless the current connection holds `operator.approvals`; -it does not expose exec, plugin, or system-agent approvals. - -Tabs use the 12-column widget grid by default. A tab containing zero or one widget can -instead use the `full` layout, which removes the widget card chrome and lets the widget -fill the tab. Switch layouts with `openclaw workspaces tabs full ` and -`openclaw workspaces tabs grid `. - -Widgets declare data through **bindings**, they never fetch on their own: - -| Binding | Resolves to | -| -------- | --------------------------------------------------------------------------------------------------------- | -| `static` | A literal value stored in the document (8 KB max). | -| `file` | A JSON, Markdown, or CSV file under `/workspaces/data/`, optionally narrowed by a JSON pointer. | -| `rpc` | One of a fixed allowlist of read-only gateway methods, resolved by the trusted Control UI. | - -The `file` binding is the simplest way to put your own numbers in a workspace: write a -JSON file into the data directory and point a `stat-card` at it. - -## Provenance - -Tabs and widgets carry a `createdBy` stamp — `user`, `system`, or `agent:` — set from -whoever made the write. It cannot be supplied by the caller, so an agent cannot label its -work as yours, and the "AI" chip on an agent-authored widget always means what it says. - -## Custom widgets - -An agent can author a real HTML widget with `workspace_widget_scaffold` (or you can, with -`openclaw workspaces widget-scaffold `). Agent-authored code is treated as hostile: - -- A scaffolded widget enters the registry as **pending**. No iframe is created, and the - asset route returns 404 for its files, until an operator approves it. -- Approval is a separate decision from editing a layout: `workspaces.widget.approve` - requires the `operator.approvals` scope, the same scope that guards exec approvals. -- An approved widget renders in an ``; -} - -export function renderIframeEmbed( - widget: WorkspaceWidget, - _value: unknown, - ctx: BuiltinWidgetContext, -): TemplateResult { - const decision = evaluateEmbedUrl(widgetProps(widget).url, { - allowExternalEmbedUrls: ctx.embed.allowExternalEmbedUrls, - }); - if (decision.status === "missing") { - return html`
- ${t("workspaces.widget.embed.missing")} -
`; - } - if (decision.status === "blocked") { - return html`
- ${decision.reason === "external" - ? t("workspaces.widget.embed.blockedExternal") - : t("workspaces.widget.embed.blockedScheme")} -
`; - } - return renderWorkspaceEmbedFrame({ - widget, - url: decision.url, - ctx, - className: "workspace-embed__frame", - testId: "workspace-embed-frame", - }); -} diff --git a/ui/src/lib/workspace/widgets/index.ts b/ui/src/lib/workspace/widgets/index.ts deleted file mode 100644 index af205e5239e..00000000000 --- a/ui/src/lib/workspace/widgets/index.ts +++ /dev/null @@ -1,43 +0,0 @@ -// Registry of L4 builtin widget renderers, consumed by the widget cell's -// dispatch (ui/src/components/workspace-widget-cell.ts). Keys are the bare kind -// (`builtin:` with the prefix stripped). Adding a builtin = add a module + -// one entry here; the plugin schema's BUILTIN_KIND_PATTERN must list the same -// names (extensions/workspaces/src/schema.ts). - -import { renderActivity } from "./activity.ts"; -import { renderAgentStatus } from "./agent-status.ts"; -import { renderChart } from "./chart.ts"; -import { renderCron } from "./cron.ts"; -import { renderCustomWidgetApprovals } from "./custom-widget-approvals.ts"; -import { renderIframeEmbed } from "./iframe-embed.ts"; -import { renderInstances } from "./instances.ts"; -import { renderMarkdown } from "./markdown.ts"; -import { renderPreview } from "./preview.ts"; -import { renderSessions } from "./sessions.ts"; -import { renderStatCard } from "./stat-card.ts"; -import { renderTable } from "./table.ts"; -import type { BuiltinWidgetRenderer } from "./types.ts"; -import { renderUsage } from "./usage.ts"; - -const BUILTIN_WIDGET_RENDERERS: Record = { - "stat-card": (widget, value) => renderStatCard(widget, value), - markdown: (widget, value) => renderMarkdown(widget, value), - table: (widget, value) => renderTable(widget, value), - "iframe-embed": renderIframeEmbed, - preview: renderPreview, - sessions: (widget, value, context) => renderSessions(widget, value, context.basePath), - usage: (widget, value) => renderUsage(widget, value), - cron: (widget, value) => renderCron(widget, value), - instances: (widget, value) => renderInstances(widget, value), - activity: (widget, value) => renderActivity(widget, value), - chart: (widget, value) => renderChart(widget, value), - "agent-status": (widget, value) => renderAgentStatus(widget, value), - "custom-widget-approvals": renderCustomWidgetApprovals, -}; - -export function getBuiltinRenderer(kind: string): BuiltinWidgetRenderer | undefined { - const name = kind.startsWith("builtin:") ? kind.slice("builtin:".length) : kind; - return BUILTIN_WIDGET_RENDERERS[name]; -} - -export type { BuiltinWidgetContext } from "./types.ts"; diff --git a/ui/src/lib/workspace/widgets/instances.ts b/ui/src/lib/workspace/widgets/instances.ts deleted file mode 100644 index df6be4497b5..00000000000 --- a/ui/src/lib/workspace/widgets/instances.ts +++ /dev/null @@ -1,101 +0,0 @@ -// builtin:instances — connected instances + health over `system-presence` -// (PresenceEntry[]; see ui/src/api/types.ts). Each entry is a live gateway/node -// presence row; the widget shows the host/instance, mode, and an idle-derived -// health dot. Thin re-implementation — the instances page owns its own masking -// and refresh state. - -import { html, nothing, type TemplateResult } from "lit"; -import { t } from "../../../i18n/index.ts"; -import { formatMs } from "../../format.ts"; -import type { WorkspaceWidget } from "../types.ts"; -import { isRecord, toFiniteNumber, widgetProps } from "./types.ts"; - -const DEFAULT_LIMIT = 8; -// A presence row idle beyond this window renders as degraded rather than live. -const HEALTHY_IDLE_SECONDS = 120; - -type InstanceModel = { - id: string; - detail: string | null; - healthy: boolean; - lastInputMs: number | null; -}; - -type InstancesModel = { - instances: InstanceModel[]; - total: number; -}; - -function instanceId(entry: Record): string { - const candidate = entry.instanceId ?? entry.host ?? entry.ip ?? entry.deviceFamily; - return typeof candidate === "string" && candidate.trim() ? candidate : ""; -} - -function instanceDetail(entry: Record): string | null { - const parts = [entry.mode, entry.platform, entry.version].filter( - (part): part is string => typeof part === "string" && part.trim().length > 0, - ); - return parts.length > 0 ? parts.join(" · ") : null; -} - -function mapInstances(widget: WorkspaceWidget, value: unknown): InstancesModel { - const raw = Array.isArray(value) - ? value - : isRecord(value) && Array.isArray(value.presence) - ? value.presence - : isRecord(value) && Array.isArray(value.nodes) - ? value.nodes - : []; - const limitProp = toFiniteNumber(widgetProps(widget).limit); - const limit = limitProp && limitProp > 0 ? Math.trunc(limitProp) : DEFAULT_LIMIT; - const records = raw.filter(isRecord); - const instances = records - .map((entry) => { - const lastInputSeconds = toFiniteNumber(entry.lastInputSeconds); - return { - id: instanceId(entry), - detail: instanceDetail(entry), - healthy: lastInputSeconds === undefined || lastInputSeconds <= HEALTHY_IDLE_SECONDS, - lastInputMs: lastInputSeconds !== undefined ? lastInputSeconds * 1000 : null, - }; - }) - .filter((entry) => entry.id) - .slice(0, limit); - return { instances, total: records.length }; -} - -export function renderInstances(widget: WorkspaceWidget, value: unknown): TemplateResult { - const model = mapInstances(widget, value); - if (model.instances.length === 0) { - return html`
- ${t("workspaces.widget.instances.empty")} -
`; - } - return html` -
    - ${model.instances.map( - (instance) => html` -
  • - - ${instance.id} - ${instance.detail - ? html`${instance.detail}` - : nothing} - ${instance.lastInputMs !== null - ? html`${t("workspaces.widget.instances.idle", { - duration: formatMs(instance.lastInputMs), - })}` - : nothing} -
  • - `, - )} -
- `; -} diff --git a/ui/src/lib/workspace/widgets/markdown.ts b/ui/src/lib/workspace/widgets/markdown.ts deleted file mode 100644 index 9e63df1ef8e..00000000000 --- a/ui/src/lib/workspace/widgets/markdown.ts +++ /dev/null @@ -1,36 +0,0 @@ -// builtin:markdown — renders a markdown body from a `content` binding (file / -// static) or `props.markdown` / `props.text`. Reuses the repo's sanitizing -// markdown util so the same allowlist/sanitizer governs workspace and chat. - -import { html, type TemplateResult } from "lit"; -import { unsafeHTML } from "lit/directives/unsafe-html.js"; -import { toSanitizedMarkdownHtml } from "../../../components/markdown.ts"; -import { t } from "../../../i18n/index.ts"; -import type { WorkspaceWidget } from "../types.ts"; -import { widgetProps } from "./types.ts"; - -function mapMarkdownSource(widget: WorkspaceWidget, value: unknown): string { - const props = widgetProps(widget); - if (typeof value === "string") { - return value; - } - if (typeof props.markdown === "string") { - return props.markdown; - } - if (typeof props.text === "string") { - return props.text; - } - return ""; -} - -export function renderMarkdown(widget: WorkspaceWidget, value: unknown): TemplateResult { - const source = mapMarkdownSource(widget, value); - if (!source.trim()) { - return html`
- ${t("workspaces.widget.markdownEmpty")} -
`; - } - return html`
- ${unsafeHTML(toSanitizedMarkdownHtml(source))} -
`; -} diff --git a/ui/src/lib/workspace/widgets/preview.ts b/ui/src/lib/workspace/widgets/preview.ts deleted file mode 100644 index c45e89f4d12..00000000000 --- a/ui/src/lib/workspace/widgets/preview.ts +++ /dev/null @@ -1,105 +0,0 @@ -// builtin:preview — an agent-authored live URL with reload and viewport controls. -// It deliberately shares iframe-embed's URL gate and sandbox ceiling: even an -// operator-trusted chat embed must not grant this builtin same-origin access. - -import { html, type TemplateResult } from "lit"; -import { createRef } from "lit/directives/ref.js"; -import { t } from "../../../i18n/index.ts"; -import type { WorkspaceWidget } from "../types.ts"; -import { evaluateEmbedUrl, renderWorkspaceEmbedFrame } from "./iframe-embed.ts"; -import type { BuiltinWidgetContext, PreviewViewport } from "./types.ts"; -import { widgetProps } from "./types.ts"; - -const PREVIEW_VIEWPORTS: readonly PreviewViewport[] = ["desktop", "tablet", "mobile"]; - -function mapPreviewViewport(widget: WorkspaceWidget): PreviewViewport { - const raw = widgetProps(widget).defaultViewport; - return raw === "tablet" || raw === "mobile" ? raw : "desktop"; -} - -function viewportClass(viewport: PreviewViewport): string { - return `workspace-preview__frame-wrap workspace-preview__frame-wrap--${viewport}`; -} - -export function renderPreview( - widget: WorkspaceWidget, - value: unknown, - ctx: BuiltinWidgetContext, -): TemplateResult { - // A declared primary binding owns the preview URL, including malformed - // values. Falling back to props would hide broken or hostile binding data. - const hasBinding = Object.hasOwn(widget.bindings ?? {}, "value"); - const rawUrl = hasBinding ? value : widgetProps(widget).url; - const decision = evaluateEmbedUrl(rawUrl, { - allowExternalEmbedUrls: ctx.embed.allowExternalEmbedUrls, - }); - if (decision.status === "missing") { - return html`
- ${t("workspaces.widget.preview.missing")} -
`; - } - if (decision.status === "blocked") { - return html`
- ${decision.reason === "external" - ? t("workspaces.widget.preview.blockedExternal") - : t("workspaces.widget.preview.blockedScheme")} -
`; - } - - const selectedViewport = ctx.preview.getViewport(widget.id, mapPreviewViewport(widget)); - const frameRef = createRef(); - - const reload = () => { - const frame = frameRef.value; - const src = frame?.getAttribute("src"); - if (frame && src !== null && src !== undefined) { - frame.setAttribute("src", src); - } - }; - - return html`
- -
- ${renderWorkspaceEmbedFrame({ - widget, - url: decision.url, - ctx, - className: "workspace-embed__frame workspace-preview__frame", - testId: "workspace-preview-frame", - frameRef, - })} -
-
`; -} diff --git a/ui/src/lib/workspace/widgets/sessions.ts b/ui/src/lib/workspace/widgets/sessions.ts deleted file mode 100644 index 751b317323f..00000000000 --- a/ui/src/lib/workspace/widgets/sessions.ts +++ /dev/null @@ -1,94 +0,0 @@ -// builtin:sessions — latest-N sessions with a live-run dot; each row links to the -// chat route for that session. Thin re-implementation over `sessions.list` -// (NOT the sessions page's welded view fns). Binding value shape: -// `{ sessions: GatewaySessionRow[] }` or a bare row array. - -import { html, nothing, type TemplateResult } from "lit"; -import type { SessionRunStatus } from "../../../api/types.ts"; -import { pathForRoute } from "../../../app-route-paths.ts"; -import { t } from "../../../i18n/index.ts"; -import { formatDateTimeMs } from "../../format.ts"; -import { isSessionRunActive } from "../../session-run-state.ts"; -import { searchForSession } from "../../sessions/navigation.ts"; -import type { WorkspaceWidget } from "../types.ts"; -import { isRecord, toFiniteNumber, widgetProps } from "./types.ts"; - -const DEFAULT_LIMIT = 6; - -type SessionsRowModel = { - key: string; - label: string; - active: boolean; - updatedAt: number | null; -}; - -type SessionsModel = { - rows: SessionsRowModel[]; - total: number; -}; - -function rowLabel(row: Record, key: string): string { - const display = row.displayName ?? row.label ?? row.subject ?? row.channel; - return typeof display === "string" && display.trim() ? display : key; -} - -function mapSessions(widget: WorkspaceWidget, value: unknown): SessionsModel { - const raw = Array.isArray(value) - ? value - : isRecord(value) && Array.isArray(value.sessions) - ? value.sessions - : []; - const limitProp = toFiniteNumber(widgetProps(widget).limit); - const limit = limitProp && limitProp > 0 ? Math.trunc(limitProp) : DEFAULT_LIMIT; - const records = raw.filter(isRecord); - const rows = records - .map((row) => { - const key = typeof row.key === "string" ? row.key : ""; - return { - key, - label: rowLabel(row, key), - active: isSessionRunActive({ - hasActiveRun: typeof row.hasActiveRun === "boolean" ? row.hasActiveRun : undefined, - status: typeof row.status === "string" ? (row.status as SessionRunStatus) : undefined, - }), - updatedAt: toFiniteNumber(row.updatedAt) ?? null, - }; - }) - .filter((row) => row.key) - .slice(0, limit); - return { rows, total: records.length }; -} - -export function renderSessions( - widget: WorkspaceWidget, - value: unknown, - basePath = "", -): TemplateResult { - const model = mapSessions(widget, value); - if (model.rows.length === 0) { - return html`
- ${t("workspaces.widget.sessions.empty")} -
`; - } - const chatPath = pathForRoute("chat", basePath); - return html` - - `; -} diff --git a/ui/src/lib/workspace/widgets/stat-card.ts b/ui/src/lib/workspace/widgets/stat-card.ts deleted file mode 100644 index 5e385bf3b13..00000000000 --- a/ui/src/lib/workspace/widgets/stat-card.ts +++ /dev/null @@ -1,80 +0,0 @@ -// builtin:stat-card — big number + label. One binding (`value`). `props.format` -// controls presentation (usd | int | percent | raw). When the binding resolves a -// structured payload (e.g. `usage.cost`), `props.metric` selects a field from it -// so a stat-card can front a rich RPC without a binding pointer (rpc bindings -// carry no pointer — see extensions/workspace schema). - -import { html, nothing, type TemplateResult } from "lit"; -import { t } from "../../../i18n/index.ts"; -import type { WorkspaceWidget } from "../types.ts"; -import { isRecord, toFiniteNumber, widgetProps } from "./types.ts"; - -type StatCardModel = { - /** The display string for the primary value, or null when unavailable. */ - display: string | null; - /** Inner label; null when it would merely repeat the widget title. */ - label: string | null; -}; - -/** Named metrics selectable from a structured binding payload via `props.metric`. */ -function selectMetric(value: unknown, metric: string): unknown { - if (!isRecord(value)) { - return undefined; - } - const totals = isRecord(value.totals) ? value.totals : undefined; - switch (metric) { - case "todayCost": - return totals?.totalCost ?? value.totalCost; - case "todayTokens": - return totals?.totalTokens ?? value.totalTokens; - default: - return value[metric]; - } -} - -function formatStatValue(value: unknown, format: unknown): string | null { - if (value === undefined || value === null) { - return null; - } - const numeric = toFiniteNumber(value); - if (format === "usd" && numeric !== undefined) { - return new Intl.NumberFormat(undefined, { style: "currency", currency: "USD" }).format(numeric); - } - if (format === "percent" && numeric !== undefined) { - return new Intl.NumberFormat(undefined, { style: "percent", maximumFractionDigits: 1 }).format( - numeric, - ); - } - if ((format === "int" || format === "integer") && numeric !== undefined) { - return new Intl.NumberFormat(undefined, { maximumFractionDigits: 0 }).format(numeric); - } - if (typeof value === "string") { - return value; - } - if (numeric !== undefined) { - return new Intl.NumberFormat(undefined).format(numeric); - } - return JSON.stringify(value); -} - -function mapStatCard(widget: WorkspaceWidget, value: unknown): StatCardModel { - const props = widgetProps(widget); - const metric = typeof props.metric === "string" ? props.metric : null; - const selected = metric ? selectMetric(value, metric) : value; - const resolved = selected !== undefined ? selected : props.value; - const label = typeof props.label === "string" ? props.label : widget.title; - // #6 nit: drop the inner label when it merely repeats the widget title — - // the cell already renders `widget.title` in the bar. - const dedupedLabel = label && label !== widget.title ? label : null; - return { display: formatStatValue(resolved, props.format), label: dedupedLabel }; -} - -export function renderStatCard(widget: WorkspaceWidget, value: unknown): TemplateResult { - const model = mapStatCard(widget, value); - return html` -
-
${model.display ?? t("workspaces.widget.stat.empty")}
- ${model.label ? html`
${model.label}
` : nothing} -
- `; -} diff --git a/ui/src/lib/workspace/widgets/table.ts b/ui/src/lib/workspace/widgets/table.ts deleted file mode 100644 index a296f701e25..00000000000 --- a/ui/src/lib/workspace/widgets/table.ts +++ /dev/null @@ -1,103 +0,0 @@ -// builtin:table — compact table over an array binding (`rows`: file JSON array / -// static / rpc). `props.columns` is a picklist of keys to show (defaults to the -// union of the first row's keys). Shows the first N rows and a "+M more" footer. - -import { html, nothing, type TemplateResult } from "lit"; -import { t } from "../../../i18n/index.ts"; -import type { WorkspaceWidget } from "../types.ts"; -import { isRecord, widgetProps } from "./types.ts"; - -const DEFAULT_ROW_LIMIT = 8; - -type TableModel = { - columns: string[]; - rows: Array>; - shown: number; - total: number; -}; - -/** Pull an array of row records out of the binding value or `props.rows`. */ -function resolveRows(widget: WorkspaceWidget, value: unknown): Array> { - const candidate = Array.isArray(value) - ? value - : isRecord(value) && Array.isArray(value.rows) - ? value.rows - : Array.isArray(widgetProps(widget).rows) - ? (widgetProps(widget).rows as unknown[]) - : []; - return candidate.filter(isRecord); -} - -function resolveColumns(widget: WorkspaceWidget, rows: Array>): string[] { - const declared = widgetProps(widget).columns; - if (Array.isArray(declared)) { - const picked = declared.filter((entry): entry is string => typeof entry === "string"); - if (picked.length > 0) { - return picked; - } - } - const firstRow = rows[0]; - return firstRow ? Object.keys(firstRow) : []; -} - -function rowLimit(widget: WorkspaceWidget): number { - const raw = widgetProps(widget).limit; - return typeof raw === "number" && Number.isFinite(raw) && raw > 0 - ? Math.min(Math.trunc(raw), 100) - : DEFAULT_ROW_LIMIT; -} - -function mapTable(widget: WorkspaceWidget, value: unknown): TableModel { - const all = resolveRows(widget, value); - const limit = rowLimit(widget); - const rows = all.slice(0, limit); - return { columns: resolveColumns(widget, rows), rows, shown: rows.length, total: all.length }; -} - -function renderCell(value: unknown): string { - if (value === null || value === undefined) { - return ""; - } - if (typeof value === "string") { - return value; - } - if (typeof value === "number" || typeof value === "boolean") { - return String(value); - } - return JSON.stringify(value); -} - -export function renderTable(widget: WorkspaceWidget, value: unknown): TemplateResult { - const model = mapTable(widget, value); - if (model.total === 0 || model.columns.length === 0) { - return html`
- ${t("workspaces.widget.table.empty")} -
`; - } - const remaining = model.total - model.shown; - return html` -
- - - - ${model.columns.map((column) => html``)} - - - - ${model.rows.map( - (row) => html` - - ${model.columns.map((column) => html``)} - - `, - )} - -
${column}
${renderCell(row[column])}
- ${remaining > 0 - ? html`` - : nothing} -
- `; -} diff --git a/ui/src/lib/workspace/widgets/types.ts b/ui/src/lib/workspace/widgets/types.ts deleted file mode 100644 index 28a2c106c96..00000000000 --- a/ui/src/lib/workspace/widgets/types.ts +++ /dev/null @@ -1,73 +0,0 @@ -// Contract for the L4 builtin widget library. Each builtin is a pure render -// function keyed by its kind (`builtin:`); the widget cell dispatches -// through the registry (`./index.ts`). Data-shape mapping lives in exported, -// separately-unit-tested `map*` helpers per widget so the RPC payload → view -// model transform is verifiable without a DOM. -// -// Renderers NEVER fetch: they receive the already-resolved primary binding -// value (the L3 view resolves the first declared binding on the page's gateway -// client) plus the widget's `props`. A binding error is surfaced by the cell's -// error boundary before a renderer runs, so renderers only see values. - -import type { TemplateResult } from "lit"; -import type { ApplicationConfigCapability } from "../../../app/config.ts"; -import type { WorkspaceWidget } from "../types.ts"; - -export type CustomWidgetApprovalDecision = "approve" | "reject"; -export type PendingCustomWidgetApproval = { - id: string; - title: string; - requestedBy: string | null; - deciding: boolean; -}; -export type CustomWidgetApprovalsSource = { - pending: PendingCustomWidgetApproval[]; - onDecide?: (item: PendingCustomWidgetApproval, decision: CustomWidgetApprovalDecision) => void; -}; - -/** Ambient context a builtin may need beyond its own binding value. */ -export type BuiltinWidgetContext = { - /** Control UI mount path used by builtins that link to another app route. */ - basePath: string; - /** Control UI embed policy — only the iframe-embed widget consumes it. */ - embed: Pick< - ApplicationConfigCapability["current"], - "embedSandboxMode" | "allowExternalEmbedUrls" - >; - /** Ephemeral preview UI state owned by the workspace view, never the document. */ - preview: { - getViewport: (widgetId: string, fallback: PreviewViewport) => PreviewViewport; - setViewport: (widgetId: string, viewport: PreviewViewport) => void; - }; - /** Pending custom-widget decisions; no exec or plugin approval queues are exposed. */ - customWidgetApprovals?: CustomWidgetApprovalsSource; -}; - -export type PreviewViewport = "desktop" | "tablet" | "mobile"; - -/** A builtin widget renderer: pure, side-effect-free, throws only on real bugs. */ -export type BuiltinWidgetRenderer = ( - widget: WorkspaceWidget, - value: unknown, - ctx: BuiltinWidgetContext, -) => TemplateResult; - -export function widgetProps(widget: WorkspaceWidget): Record { - return widget.props ?? {}; -} - -export function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -/** Coerce a possibly-string numeric field to a finite number, else undefined. */ -export function toFiniteNumber(value: unknown): number | undefined { - if (typeof value === "number") { - return Number.isFinite(value) ? value : undefined; - } - if (typeof value === "string" && value.trim()) { - const parsed = Number(value); - return Number.isFinite(parsed) ? parsed : undefined; - } - return undefined; -} diff --git a/ui/src/lib/workspace/widgets/usage.ts b/ui/src/lib/workspace/widgets/usage.ts deleted file mode 100644 index 517b3cdef45..00000000000 --- a/ui/src/lib/workspace/widgets/usage.ts +++ /dev/null @@ -1,38 +0,0 @@ -// builtin:usage — a today/window cost + tokens mini-summary over `usage.cost`. -// Binding value shape: `{ totals: CostUsageTotals }` (see -// src/infra/session-cost-usage.types.ts). Thin re-implementation — the usage -// page's own view fns are welded to its filter state. - -import { html, type TemplateResult } from "lit"; -import { t } from "../../../i18n/index.ts"; -import { formatCost, formatTokens } from "../../format.ts"; -import type { WorkspaceWidget } from "../types.ts"; -import { isRecord, toFiniteNumber } from "./types.ts"; - -type UsageModel = { - cost: number; - tokens: number; -}; - -function mapUsage(_widget: WorkspaceWidget, value: unknown): UsageModel { - const totals = isRecord(value) && isRecord(value.totals) ? value.totals : {}; - const cost = toFiniteNumber(totals.totalCost) ?? 0; - const tokens = toFiniteNumber(totals.totalTokens) ?? 0; - return { cost, tokens }; -} - -export function renderUsage(widget: WorkspaceWidget, value: unknown): TemplateResult { - const model = mapUsage(widget, value); - return html` -
-
-
${formatCost(model.cost)}
-
${t("workspaces.widget.usage.cost")}
-
-
-
${formatTokens(model.tokens)}
-
${t("workspaces.widget.usage.tokens")}
-
-
- `; -} diff --git a/ui/src/lib/workspace/widgets/widgets.test.ts b/ui/src/lib/workspace/widgets/widgets.test.ts deleted file mode 100644 index a12a9e91fa5..00000000000 --- a/ui/src/lib/workspace/widgets/widgets.test.ts +++ /dev/null @@ -1,585 +0,0 @@ -// Focused renderer tests for builtin data shapes and empty/error affordances. - -import { render } from "lit"; -import { describe, expect, it, vi } from "vitest"; -import type { WorkspaceWidget } from "../types.ts"; -import { renderActivity } from "./activity.ts"; -import { renderAgentStatus } from "./agent-status.ts"; -import { renderChart } from "./chart.ts"; -import { renderCron } from "./cron.ts"; -import { - buildCustomWidgetApprovalsSource, - renderCustomWidgetApprovals, -} from "./custom-widget-approvals.ts"; -import { renderIframeEmbed } from "./iframe-embed.ts"; -import { renderInstances } from "./instances.ts"; -import { renderMarkdown } from "./markdown.ts"; -import { renderPreview } from "./preview.ts"; -import { renderSessions } from "./sessions.ts"; -import { renderStatCard } from "./stat-card.ts"; -import { renderTable } from "./table.ts"; -import type { BuiltinWidgetContext } from "./types.ts"; -import { renderUsage } from "./usage.ts"; - -function widget(overrides: Partial = {}): WorkspaceWidget { - return { - id: "w1", - kind: "builtin:stat-card", - title: "Widget", - grid: { x: 0, y: 0, w: 4, h: 2 }, - collapsed: false, - ...overrides, - }; -} - -function renderToContainer(template: unknown): HTMLElement { - const container = document.createElement("div"); - render(template as never, container); - return container; -} - -const STRICT_EMBED: BuiltinWidgetContext = { - basePath: "", - embed: { embedSandboxMode: "strict", allowExternalEmbedUrls: false }, - preview: { getViewport: (_widgetId, fallback) => fallback, setViewport: vi.fn() }, -}; - -describe("agent-status mapping", () => { - it("maps only keyed sessions and clamps goal progress", () => { - const container = renderToContainer( - renderAgentStatus(widget({ props: { limit: 2 } }), { - sessions: [ - { - key: "agent:one", - displayName: "One", - hasActiveRun: true, - goal: { objective: "Ship the workspace", tokensUsed: 125, tokenBudget: 100 }, - }, - { key: "agent:two", status: "idle" }, - { displayName: "missing key" }, - ], - }), - ); - expect(container.querySelectorAll(".workspace-list__row")).toHaveLength(2); - expect(container.textContent).toContain("Ship the workspace"); - expect(container.textContent).toContain("100"); - expect(container.textContent).not.toContain("missing key"); - }); - - it("renders accessible status text and an empty state", () => { - const populated = renderToContainer( - renderAgentStatus(widget(), { sessions: [{ key: "agent:one", hasActiveRun: true }] }), - ); - expect( - populated.querySelector("[data-test-id='workspace-agent-status']")?.textContent, - ).toContain("Busy"); - const empty = renderToContainer(renderAgentStatus(widget(), { sessions: [] })); - expect(empty.querySelector(".workspace-widget__placeholder")).not.toBeNull(); - }); -}); - -describe("custom-widget-approvals mapping", () => { - it("exposes only pending custom-widget registry entries", () => { - const decisions: Array<[string, "approved" | "rejected"]> = []; - const source = buildCustomWidgetApprovalsSource( - { - schemaVersion: 1, - workspaceVersion: 3, - tabs: [], - prefs: { tabOrder: [] }, - widgetsRegistry: { - pending: { status: "pending", createdBy: "agent:builder" }, - approved: { status: "approved", createdBy: "agent:builder" }, - }, - }, - new Set(), - (name, decision) => decisions.push([name, decision]), - ); - expect(source.pending).toEqual([ - { id: "pending", title: "pending", requestedBy: "builder", deciding: false }, - ]); - source.onDecide?.(source.pending[0]!, "reject"); - expect(decisions).toEqual([["pending", "rejected"]]); - }); - - it("limits rows and renders permission-aware decision controls", () => { - const source = { - pending: [ - { id: "one", title: "one", requestedBy: null, deciding: false }, - { id: "two", title: "two", requestedBy: "builder", deciding: false }, - ], - onDecide: () => undefined, - }; - const container = renderToContainer( - renderCustomWidgetApprovals(widget({ props: { limit: 1 } }), undefined, { - ...STRICT_EMBED, - customWidgetApprovals: source, - }), - ); - expect(container.querySelectorAll(".workspace-list__row")).toHaveLength(1); - expect(container.querySelectorAll("button")).toHaveLength(2); - expect(container.querySelector("button")?.disabled).toBe(false); - expect(container.textContent).toContain("Approve"); - expect(container.textContent).toContain("Reject"); - - const restricted = renderToContainer( - renderCustomWidgetApprovals(widget(), undefined, { - ...STRICT_EMBED, - customWidgetApprovals: { pending: source.pending }, - }), - ); - expect( - Array.from(restricted.querySelectorAll("button")).every((button) => button.disabled), - ).toBe(true); - expect(restricted.textContent).toContain("Approval permission required"); - }); -}); - -describe("stat-card mapping", () => { - it("renders the value and omits a duplicate label", () => { - const container = renderToContainer( - renderStatCard(widget({ title: "Cost", props: { label: "Cost", format: "usd" } }), 9), - ); - expect(container.querySelector(".workspace-stat__value")?.textContent).toContain("$9"); - expect(container.querySelector(".workspace-stat__label")).toBeNull(); - }); -}); - -describe("markdown mapping", () => { - it("renders an empty state when there is no content", () => { - const container = renderToContainer(renderMarkdown(widget(), "")); - expect(container.querySelector(".workspace-widget__placeholder")).not.toBeNull(); - }); -}); - -describe("table mapping", () => { - const rows = [ - { name: "a", cost: 1 }, - { name: "b", cost: 2 }, - { name: "c", cost: 3 }, - ]; - - it("accepts { rows } payloads and renders a +N more footer", () => { - const container = renderToContainer(renderTable(widget({ props: { limit: 2 } }), { rows })); - expect(container.querySelector(".workspace-table__footer")?.textContent).toContain("1"); - }); - - it("renders an empty state for no rows", () => { - const container = renderToContainer(renderTable(widget(), [])); - expect(container.querySelector(".workspace-widget__placeholder")).not.toBeNull(); - }); -}); - -describe("sessions mapping", () => { - it("renders a link per session and an empty state", () => { - const populated = renderToContainer( - renderSessions(widget(), { sessions: [{ key: "main:1", displayName: "One" }] }, "/openclaw"), - ); - expect(populated.querySelector(".workspace-list__link")?.getAttribute("href")).toBe( - "/openclaw/chat?session=main%3A1", - ); - const empty = renderToContainer(renderSessions(widget(), { sessions: [] })); - expect(empty.querySelector(".workspace-widget__placeholder")).not.toBeNull(); - }); -}); - -describe("usage mapping", () => { - it("renders both cost and token metrics", () => { - const container = renderToContainer( - renderUsage(widget(), { totals: { totalCost: 5, totalTokens: 2000 } }), - ); - const values = [...container.querySelectorAll(".workspace-usage__value")].map( - (n) => n.textContent, - ); - expect(values).toHaveLength(2); - }); -}); - -describe("cron mapping", () => { - it("renders an empty state without jobs", () => { - const container = renderToContainer(renderCron(widget(), { jobs: [] })); - expect(container.querySelector(".workspace-widget__placeholder")).not.toBeNull(); - }); -}); - -describe("instances mapping", () => { - it("accepts a { presence } wrapper and renders an empty state", () => { - const populated = renderToContainer( - renderInstances(widget(), { presence: [{ instanceId: "gw-1" }] }), - ); - expect(populated.querySelector(".workspace-instances")).not.toBeNull(); - const empty = renderToContainer(renderInstances(widget(), [])); - expect(empty.querySelector(".workspace-widget__placeholder")).not.toBeNull(); - }); -}); - -describe("activity mapping", () => { - it("renders an empty state for no entries", () => { - const container = renderToContainer(renderActivity(widget(), { entries: [] })); - expect(container.querySelector(".workspace-widget__placeholder")).not.toBeNull(); - }); -}); - -describe("chart mapping", () => { - it("renders single-point and constant series visibly", () => { - const line = renderToContainer(renderChart(widget({ props: { type: "line" } }), [5])); - expect(line.querySelector(".workspace-chart__point")).not.toBeNull(); - - const bars = renderToContainer(renderChart(widget({ props: { type: "bar" } }), [5, 5])); - expect( - [...bars.querySelectorAll(".workspace-chart__bars rect")].every( - (bar) => Number(bar.getAttribute("height")) > 0.5, - ), - ).toBe(true); - - const zeroBars = renderToContainer(renderChart(widget({ props: { type: "bar" } }), [0, 0])); - expect( - [...zeroBars.querySelectorAll(".workspace-chart__bars rect")].every( - (bar) => Number(bar.getAttribute("height")) === 0, - ), - ).toBe(true); - }); - - it("keeps extreme finite ranges within valid SVG coordinates", () => { - const container = renderToContainer( - renderChart(widget(), [-Number.MAX_VALUE, Number.MAX_VALUE]), - ); - const points = container.querySelector("polyline")?.getAttribute("points") ?? ""; - expect(points).not.toMatch(/NaN|Infinity/); - - const constant = renderToContainer(renderChart(widget(), [Number.MAX_VALUE])); - expect(constant.querySelector("circle")?.outerHTML).not.toMatch(/NaN|Infinity/); - - const gauge = renderToContainer( - renderChart(widget({ props: { type: "gauge" } }), [-Number.MAX_VALUE, Number.MAX_VALUE]), - ); - expect(gauge.querySelector(".workspace-chart__gauge")?.outerHTML).not.toMatch(/NaN|Infinity/); - - const bounded = renderToContainer(renderChart(widget({ props: { min: 0, max: 1 } }), [1e308])); - expect(bounded.querySelector("svg")?.outerHTML).not.toMatch(/NaN|Infinity/); - - const subnormalMin = renderToContainer( - renderChart(widget({ props: { min: Number.MIN_VALUE } }), [Number.MIN_VALUE]), - ); - expect(subnormalMin.querySelector("svg")?.outerHTML).not.toMatch(/NaN|Infinity/); - - const subnormalMax = renderToContainer( - renderChart(widget({ props: { max: -Number.MIN_VALUE } }), [-Number.MIN_VALUE]), - ); - expect(subnormalMax.querySelector("svg")?.outerHTML).not.toMatch(/NaN|Infinity/); - - for (const [props, values] of [ - [{ min: 10 }, [1, 2]], - [{ max: -10 }, [-2, -1]], - ] as const) { - const oneSided = renderToContainer(renderChart(widget({ props }), values)); - expect(oneSided.querySelector("svg")?.outerHTML).not.toMatch(/NaN|Infinity/); - expect(oneSided.querySelector('[data-test-id="workspace-chart-error"]')).toBeNull(); - } - }); - - it("announces data extrema rather than configured axis bounds", () => { - const container = renderToContainer( - renderChart(widget({ title: "Revenue", props: { min: 0, max: 10 } }), [2, 7]), - ); - expect(container.querySelector("svg")?.getAttribute("aria-label")).toContain( - "ranging from 2 to 7", - ); - }); - - it("renders responsive accessible line, bar, area, sparkline, and gauge charts", () => { - for (const type of ["line", "bar", "area", "sparkline", "gauge"] as const) { - const container = renderToContainer( - renderChart(widget({ title: "Revenue", props: { type, min: 0, max: 10 } }), [2, 7]), - ); - const svg = container.querySelector('[data-test-id="workspace-chart"]'); - expect(svg?.getAttribute("role")).toBe("img"); - expect(svg?.getAttribute("aria-label")).toContain("Revenue"); - expect(svg?.getAttribute("viewBox")).toBe("0 0 100 40"); - expect(container.querySelector(`.workspace-chart--${type}`)).not.toBeNull(); - } - }); - - it("renders localized empty and invalid-data states without an svg", () => { - const empty = renderToContainer(renderChart(widget(), [])); - expect(empty.querySelector('[data-test-id="workspace-chart-empty"]')).not.toBeNull(); - expect(empty.querySelector("svg")).toBeNull(); - - for (const props of [{ min: "bad" }, { min: 10, max: 2 }, { min: 4, max: 4 }]) { - const invalidEmpty = renderToContainer(renderChart(widget({ props }), [])); - expect(invalidEmpty.querySelector('[data-test-id="workspace-chart-error"]')).not.toBeNull(); - expect(invalidEmpty.querySelector("svg")).toBeNull(); - } - - const invalid = renderToContainer(renderChart(widget(), [1, "bad"])); - expect(invalid.querySelector('[data-test-id="workspace-chart-error"]')).not.toBeNull(); - expect(invalid.querySelector("svg")).toBeNull(); - - for (const [configuredWidget, value] of [ - [widget({ props: { type: "pie" } }), [1]], - [widget({ props: { type: null } }), [1]], - [widget({ props: { min: 10, max: 2 } }), [4]], - [widget(), { points: [{ label: "missing value" }] }], - [widget(), { points: Array.from({ length: 501 }, () => 1) }], - ] as const) { - const error = renderToContainer(renderChart(configuredWidget, value)); - expect(error.querySelector('[data-test-id="workspace-chart-error"]')).not.toBeNull(); - expect(error.querySelector("svg")).toBeNull(); - } - }); - - it("accepts the documented point shapes through the 500-point limit", () => { - const wrapped = renderToContainer( - renderChart(widget(), { points: [1, { y: 2 }, { value: 3 }] }), - ); - expect(wrapped.querySelector('[data-test-id="workspace-chart"]')).not.toBeNull(); - - const capped = renderToContainer( - renderChart( - widget(), - Array.from({ length: 500 }, () => 1), - ), - ); - expect(capped.querySelector('[data-test-id="workspace-chart"]')).not.toBeNull(); - - const denseBars = renderToContainer( - renderChart( - widget({ props: { type: "bar" } }), - Array.from({ length: 500 }, (_, index) => index % 2), - ), - ); - const slotWidth = (100 - 2 * 2) / 500; - expect( - [...denseBars.querySelectorAll("rect")].every( - (bar) => Number(bar.getAttribute("width")) <= slotWidth, - ), - ).toBe(true); - }); -}); - -describe("iframe-embed render × sandbox mode", () => { - it("emits a sandboxed frame for an allowed URL (strict → empty sandbox attr)", () => { - const container = renderToContainer( - renderIframeEmbed(widget({ props: { url: "/preview" } }), null, STRICT_EMBED), - ); - const frame = container.querySelector( - '[data-test-id="workspace-embed-frame"]', - ); - expect(frame).not.toBeNull(); - expect(frame?.getAttribute("sandbox")).toBe(""); - }); - - it("scripts mode grants allow-scripts", () => { - const container = renderToContainer( - renderIframeEmbed(widget({ props: { url: "/preview" } }), null, { - basePath: "", - embed: { embedSandboxMode: "scripts", allowExternalEmbedUrls: false }, - preview: STRICT_EMBED.preview, - }), - ); - const frame = container.querySelector( - '[data-test-id="workspace-embed-frame"]', - ); - expect(frame?.getAttribute("sandbox")).toBe("allow-scripts"); - }); - - it("never grants allow-same-origin, even when the operator trusts chat embeds", () => { - // `props.url` is agent-authored and a builtin needs no approval, so a - // same-origin scripted frame would hand the widget the parent's origin. - const container = renderToContainer( - renderIframeEmbed(widget({ props: { url: "/preview" } }), null, { - basePath: "", - embed: { embedSandboxMode: "trusted", allowExternalEmbedUrls: false }, - preview: STRICT_EMBED.preview, - }), - ); - const frame = container.querySelector( - '[data-test-id="workspace-embed-frame"]', - ); - expect(frame?.getAttribute("sandbox")).toBe("allow-scripts"); - expect(frame?.getAttribute("sandbox")).not.toContain("allow-same-origin"); - }); - - it("shows a blocked placeholder for an external URL under strict policy", () => { - const container = renderToContainer( - renderIframeEmbed(widget({ props: { url: "https://evil.example" } }), null, STRICT_EMBED), - ); - expect(container.querySelector('[data-test-id="workspace-embed-blocked"]')).not.toBeNull(); - expect(container.querySelector('[data-test-id="workspace-embed-frame"]')).toBeNull(); - }); -}); - -describe("preview widget", () => { - it("renders its URL with the workspace embed sandbox ceiling", () => { - const container = renderToContainer( - renderPreview(widget({ props: { url: "/preview" } }), undefined, { - basePath: "", - embed: { embedSandboxMode: "trusted", allowExternalEmbedUrls: false }, - preview: STRICT_EMBED.preview, - }), - ); - const frame = container.querySelector( - '[data-test-id="workspace-preview-frame"]', - ); - expect(frame?.getAttribute("src")).toBe("/preview"); - expect(frame?.getAttribute("sandbox")).toBe("allow-scripts"); - expect(frame?.getAttribute("sandbox")).not.toContain("allow-same-origin"); - expect(frame?.getAttribute("referrerpolicy")).toBe("no-referrer"); - }); - - it("reloads the mounted frame without reading its content window", () => { - const container = renderToContainer( - renderPreview(widget({ props: { url: "/preview" } }), undefined, STRICT_EMBED), - ); - const frame = container.querySelector( - '[data-test-id="workspace-preview-frame"]', - ); - expect(frame).not.toBeNull(); - const setAttribute = vi.spyOn(frame!, "setAttribute"); - container - .querySelector('[data-test-id="workspace-preview-reload"]') - ?.click(); - expect(setAttribute).toHaveBeenCalledWith("src", "/preview"); - }); - - it("exposes labeled controls and updates the selected viewport state", () => { - let selected: "desktop" | "tablet" | "mobile" | undefined; - const context: BuiltinWidgetContext = { - ...STRICT_EMBED, - preview: { - getViewport: (_widgetId, fallback) => selected ?? fallback, - setViewport: (_widgetId, viewport) => { - selected = viewport; - }, - }, - }; - const previewWidget = widget({ props: { url: "/preview", defaultViewport: "tablet" } }); - const container = renderToContainer(renderPreview(previewWidget, undefined, context)); - expect(container.querySelector('[role="toolbar"]')?.getAttribute("aria-label")).toBeTruthy(); - expect(container.querySelector('[role="group"]')?.getAttribute("aria-label")).toBeTruthy(); - const tablet = container.querySelector( - '[data-test-id="workspace-preview-viewport-tablet"]', - ); - const mobile = container.querySelector( - '[data-test-id="workspace-preview-viewport-mobile"]', - ); - expect(tablet?.getAttribute("aria-pressed")).toBe("true"); - expect(mobile?.getAttribute("aria-pressed")).toBe("false"); - mobile?.click(); - expect(selected).toBe("mobile"); - render(renderPreview(previewWidget, undefined, context), container); - expect( - container - .querySelector('[data-test-id="workspace-preview-viewport-tablet"]') - ?.getAttribute("aria-pressed"), - ).toBe("false"); - expect( - container - .querySelector('[data-test-id="workspace-preview-viewport-mobile"]') - ?.getAttribute("aria-pressed"), - ).toBe("true"); - expect(container.querySelector(".workspace-preview__frame-wrap")?.className).toContain( - "workspace-preview__frame-wrap--mobile", - ); - }); - - it.each(["bogus", 42])("defaults malformed viewport prop %j to desktop", (defaultViewport) => { - const container = renderToContainer( - renderPreview( - widget({ props: { url: "/preview", defaultViewport } }), - undefined, - STRICT_EMBED, - ), - ); - - expect(container.querySelector(".workspace-preview__frame-wrap")?.className).toContain( - "workspace-preview__frame-wrap--desktop", - ); - }); - - it("uses a bound URL instead of props and does not hide malformed binding data", () => { - const bound = renderToContainer( - renderPreview( - widget({ - props: { url: "/stale" }, - bindings: { value: { source: "static", value: "/bound" } }, - }), - "/bound", - STRICT_EMBED, - ), - ); - expect( - bound.querySelector('[data-test-id="workspace-preview-frame"]')?.getAttribute("src"), - ).toBe("/bound"); - - const malformed = renderToContainer( - renderPreview( - widget({ - props: { url: "/fallback" }, - bindings: { value: { source: "static", value: 42 } }, - }), - 42, - STRICT_EMBED, - ), - ); - expect(malformed.querySelector('[data-test-id="workspace-preview-frame"]')).toBeNull(); - - const missing = renderToContainer( - renderPreview( - widget({ - props: { url: "/fallback" }, - bindings: { value: { source: "static" } }, - }), - undefined, - STRICT_EMBED, - ), - ); - expect(missing.querySelector('[data-test-id="workspace-preview-frame"]')).toBeNull(); - - const unrelated = renderToContainer( - renderPreview( - widget({ - props: { url: "/configured" }, - bindings: { other: { source: "static", value: "/ignored" } }, - }), - "/ignored", - STRICT_EMBED, - ), - ); - expect( - unrelated.querySelector('[data-test-id="workspace-preview-frame"]')?.getAttribute("src"), - ).toBe("/configured"); - }); - - it("allows policy-approved external URLs and blocks external or unsafe URLs by default", () => { - const allowed = renderToContainer( - renderPreview( - widget({ - bindings: { - value: { source: "static", value: "https://preview.example" }, - }, - }), - "https://preview.example", - { - ...STRICT_EMBED, - embed: { embedSandboxMode: "strict", allowExternalEmbedUrls: true }, - }, - ), - ); - expect( - allowed.querySelector('[data-test-id="workspace-preview-frame"]')?.getAttribute("src"), - ).toBe("https://preview.example"); - - for (const value of [undefined, 42, "https://evil.example", "javascript:alert(1)"]) { - const container = renderToContainer( - renderPreview( - widget({ bindings: { value: { source: "static", value } } }), - value, - STRICT_EMBED, - ), - ); - expect(container.querySelector('[data-test-id="workspace-preview-frame"]')).toBeNull(); - expect(container.querySelector(".workspace-widget__placeholder")).not.toBeNull(); - } - }); -}); diff --git a/ui/src/pages/plugin/plugin-page.test.ts b/ui/src/pages/plugin/plugin-page.test.ts index 40488505d4d..af0397233db 100644 --- a/ui/src/pages/plugin/plugin-page.test.ts +++ b/ui/src/pages/plugin/plugin-page.test.ts @@ -627,16 +627,19 @@ describe("PluginPage", () => { }); it("does not install an earlier bundled view after switching away and back", async () => { - const firstWorkspaceLoad = deferred(); - const currentWorkspaceLoad = deferred(); - const logbookLoad = deferred(); + const firstLogbookLoad = deferred(); + const currentLogbookLoad = deferred(); const hello: GatewayHelloOk = { type: "hello-ok", protocol: 3, auth: { role: "operator", scopes: ["operator.write"] }, controlUiTabs: [ - { pluginId: "workspaces", id: "workspaces", label: "Workspaces" }, { pluginId: "logbook", id: "logbook", label: "Logbook" }, + { + pluginId: "external-plugin", + id: "panel", + label: "External panel", + }, ], }; const snapshot: ApplicationGatewaySnapshot = { @@ -651,34 +654,32 @@ describe("PluginPage", () => { }; const page = document.createElement(deferredPluginPageTag) as DeferredPluginPage; page.loads = new Map([ - ["workspaces/workspaces", [firstWorkspaceLoad.promise, currentWorkspaceLoad.promise]], - ["logbook/logbook", [logbookLoad.promise]], + ["logbook/logbook", [firstLogbookLoad.promise, currentLogbookLoad.promise]], ]); - page.pluginId = "workspaces"; - page.tabId = "workspaces"; + page.pluginId = "logbook"; + page.tabId = "logbook"; (page as unknown as { context: ApplicationContext }).context = { gateway: { snapshot, subscribe: () => () => undefined }, } as unknown as ApplicationContext; document.body.append(page); try { + await page.updateComplete; + page.pluginId = "external-plugin"; + page.tabId = "panel"; await page.updateComplete; page.pluginId = "logbook"; page.tabId = "logbook"; await page.updateComplete; - page.pluginId = "workspaces"; - page.tabId = "workspaces"; - await page.updateComplete; - currentWorkspaceLoad.resolve({ render: () => "current workspace view", stop: vi.fn() }); - await waitForFast(() => expect(page.textContent).toContain("current workspace view")); + currentLogbookLoad.resolve({ render: () => "current Logbook view", stop: vi.fn() }); + await waitForFast(() => expect(page.textContent).toContain("current Logbook view")); - firstWorkspaceLoad.resolve({ render: () => "stale workspace view", stop: vi.fn() }); + firstLogbookLoad.resolve({ render: () => "stale Logbook view", stop: vi.fn() }); await Promise.resolve(); await page.updateComplete; - expect(page.textContent).not.toContain("stale workspace view"); - expect(page.textContent).toContain("current workspace view"); - logbookLoad.resolve({ render: () => "stale Logbook view", stop: vi.fn() }); + expect(page.textContent).not.toContain("stale Logbook view"); + expect(page.textContent).toContain("current Logbook view"); } finally { page.remove(); } diff --git a/ui/src/pages/plugin/plugin-page.ts b/ui/src/pages/plugin/plugin-page.ts index 9907ea3ea38..26bca91cc2d 100644 --- a/ui/src/pages/plugin/plugin-page.ts +++ b/ui/src/pages/plugin/plugin-page.ts @@ -72,13 +72,6 @@ const EXTERNAL_AUTH_PROBE_TIMEOUT_MS = 5_000; // Keyed by pluginId/tabId: tab ids are only unique within their plugin. const BUNDLED_TAB_VIEWS: Record Promise> = { - "workspaces/workspaces": async () => { - const [{ renderWorkspace }, { stopWorkspace }] = await Promise.all([ - import("./workspace-view.ts"), - import("./workspace-controller.ts"), - ]); - return { render: renderWorkspace, stop: stopWorkspace }; - }, "logbook/logbook": async () => { const [{ renderLogbook }, { stopLogbookPolling }] = await Promise.all([ import("./logbook-view.ts"), @@ -552,8 +545,6 @@ export class PluginPage extends OpenClawLightDomContentsElement { return nothing; } const snapshot = context.gateway.snapshot; - // Config may be absent in unit harnesses; the Workspaces view defaults the - // embed policy to strict when `embed` is omitted. const config = context.config?.current; return this.bundledView.render({ host: this.bundledViewHost, diff --git a/ui/src/pages/plugin/workspace-controller.test.ts b/ui/src/pages/plugin/workspace-controller.test.ts deleted file mode 100644 index 2e798af55c4..00000000000 --- a/ui/src/pages/plugin/workspace-controller.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import type { GatewayBrowserClient } from "../../api/gateway.ts"; -import { getWorkspaceState, subscribeToWorkspaceEvents } from "../../lib/workspace/index.ts"; -import { stopWorkspace } from "./workspace-controller.ts"; - -describe("workspace controller", () => { - it("stops the live-update subscription for its host", () => { - const host = {}; - const state = getWorkspaceState(host); - const unsubscribe = vi.fn(); - const client = { - request: vi.fn(), - addEventListener: vi.fn(() => unsubscribe), - } as unknown as GatewayBrowserClient; - subscribeToWorkspaceEvents(host, state, client); - stopWorkspace(host); - expect(unsubscribe).toHaveBeenCalledTimes(1); - }); - - it("is a no-op for a host that never subscribed", () => { - expect(() => stopWorkspace({})).not.toThrow(); - }); -}); diff --git a/ui/src/pages/plugin/workspace-controller.ts b/ui/src/pages/plugin/workspace-controller.ts deleted file mode 100644 index 1ec76c00d80..00000000000 --- a/ui/src/pages/plugin/workspace-controller.ts +++ /dev/null @@ -1,15 +0,0 @@ -// Control UI controller for the Workspaces tab: thin lifecycle glue. All data and -// mutation logic lives in `lib/workspace`; this module exposes only the `stop` -// hook the bundled-view registry (`plugin-page.ts`) wires into tab-switch and -// disconnect cleanup (matches logbook-controller's shape). - -import { stopWorkspace as stopWorkspaceLifecycle } from "../../lib/workspace/index.ts"; -import { stopWorkspaceView } from "./workspace-view.ts"; - -/** Tears down the live-update subscription; wired into tab-switch + disconnect. */ -export function stopWorkspace(host: object): void { - stopWorkspaceLifecycle(host); - // Drop the view-owned menu-dismiss document listeners (#3) so a tab-switch or - // disconnect never leaves a stray global pointerdown/keydown handler. - stopWorkspaceView(host); -} diff --git a/ui/src/pages/plugin/workspace-view.test.ts b/ui/src/pages/plugin/workspace-view.test.ts deleted file mode 100644 index 549a5f01cbb..00000000000 --- a/ui/src/pages/plugin/workspace-view.test.ts +++ /dev/null @@ -1,519 +0,0 @@ -import { render } from "lit"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import type { GatewayBrowserClient } from "../../api/gateway.ts"; -import { getWorkspaceState } from "../../lib/workspace/index.ts"; -import { waitForFast } from "../../test-helpers/wait-for.ts"; -import { stopWorkspace } from "./workspace-controller.ts"; -import { renderWorkspace as renderWorkspaceView } from "./workspace-view.ts"; - -function renderWorkspace( - props: Omit[0], "onRequestUpdate">, -) { - return renderWorkspaceView({ ...props, onRequestUpdate: () => undefined }); -} - -function renderView(host: object): HTMLElement { - const container = document.createElement("div"); - // The view queries `.workspace-grid` on the host for grid metrics. - const el = container as unknown as object; - render(renderWorkspace({ host: el, client: null, connected: false }), container); - render(renderWorkspace({ host, client: null, connected: false }), container); - return container; -} - -const doc = { - schemaVersion: 1, - workspaceVersion: 1, - tabs: [ - { - slug: "main", - title: "Main", - hidden: false, - widgets: [ - { - id: "w1", - kind: "builtin:markdown", - title: "Notes", - grid: { x: 0, y: 0, w: 6, h: 2 }, - collapsed: false, - props: { markdown: "hello" }, - }, - ], - }, - { slug: "hidden-one", title: "Hidden", hidden: true, widgets: [] }, - { slug: "empty", title: "Empty", hidden: false, widgets: [] }, - ], - widgetsRegistry: {}, - prefs: { tabOrder: ["main", "empty", "hidden-one"] }, -}; - -describe("renderWorkspace", () => { - afterEach(() => { - window.history.replaceState({}, "", "/"); - }); - - it("shows the onboarding empty state with no tabs", () => { - const host = {}; - const state = getWorkspaceState(host); - state.loaded = true; - state.workspace = { - schemaVersion: 1, - workspaceVersion: 1, - tabs: [], - widgetsRegistry: {}, - prefs: { tabOrder: [] }, - }; - const container = renderView(host); - expect(container.querySelector('[data-test-id="workspace-empty"]')).not.toBeNull(); - }); - - it("renders the tab strip with visible tabs and a hidden overflow", () => { - const host = {}; - const state = getWorkspaceState(host); - state.loaded = true; - state.workspace = doc; - state.activeSlug = "main"; - const container = renderView(host); - const tabs = container.querySelectorAll('[data-test-id="workspace-tab"]'); - expect(tabs.length).toBe(2); // main + empty (hidden-one is in overflow) - expect(container.querySelector(".workspace-tabs__hidden")).not.toBeNull(); - // Active tab's widget grid renders. - expect(container.querySelector('[data-test-id="workspace-grid"]')).not.toBeNull(); - }); - - it("renders the empty-tab hint for a tab with no widgets", () => { - const host = {}; - const state = getWorkspaceState(host); - state.loaded = true; - state.workspace = doc; - state.activeSlug = "empty"; - const container = renderView(host); - expect(container.querySelector('[data-test-id="workspace-empty-tab"]')).not.toBeNull(); - }); - - it("renders the first widget full bleed without grid chrome", () => { - const host = {}; - const state = getWorkspaceState(host); - state.loaded = true; - state.workspace = structuredClone(doc); - state.workspace.tabs[0]!.layout = "full"; - state.activeSlug = "main"; - const container = renderView(host); - expect(container.querySelector('[data-test-id="workspace-fullbleed"]')).not.toBeNull(); - expect(container.querySelector('[data-test-id="workspace-grid"]')).toBeNull(); - expect(container.textContent).toContain("hello"); - }); - - it("surfaces an action error toast", () => { - const host = {}; - const state = getWorkspaceState(host); - state.loaded = true; - state.workspace = doc; - state.activeSlug = "main"; - state.actionError = "move failed"; - const container = renderView(host); - expect(container.querySelector(".workspace__toast")?.textContent).toContain("move failed"); - }); - - it("honors workspace deep links and updates them from tab navigation", () => { - window.history.replaceState({}, "", "/plugin?plugin=workspaces&id=workspaces&ws=empty"); - const host = document.createElement("div"); - document.body.append(host); - const state = getWorkspaceState(host); - state.loaded = true; - state.workspace = doc; - state.activeSlug = "main"; - const onPopState = vi.fn(); - window.addEventListener("popstate", onPopState); - - try { - render(renderWorkspace({ host, client: null, connected: false }), host); - expect(state.activeSlug).toBe("empty"); - host - .querySelector("wa-tab-group") - ?.dispatchEvent(new CustomEvent("wa-tab-show", { detail: { name: "main" } })); - expect(new URLSearchParams(window.location.search).get("ws")).toBe("main"); - expect(onPopState).toHaveBeenCalledOnce(); - } finally { - window.removeEventListener("popstate", onPopState); - stopWorkspace(host); - host.remove(); - } - }); - - it("cancels a removed deep link before the initial workspace finishes loading", async () => { - window.history.replaceState({}, "", "/plugin?plugin=workspaces&id=workspaces&ws=empty"); - const host = document.createElement("div"); - document.body.append(host); - let resolveWorkspace!: (value: unknown) => void; - const request = vi.fn( - () => - new Promise((resolve) => { - resolveWorkspace = resolve; - }), - ); - const client = { - request, - addEventListener: vi.fn(() => () => {}), - } as unknown as GatewayBrowserClient; - const state = getWorkspaceState(host); - - try { - render(renderWorkspace({ host, client, connected: true }), host); - expect(request).toHaveBeenCalledOnce(); - - window.history.replaceState({}, "", "/plugin?plugin=workspaces&id=workspaces"); - render(renderWorkspace({ host, client, connected: true }), host); - - resolveWorkspace({ doc, workspaceVersion: doc.workspaceVersion }); - await waitForFast(() => expect(state.loaded).toBe(true)); - expect(state.activeSlug).toBe("main"); - } finally { - stopWorkspace(host); - host.remove(); - } - }); - - it("discards a stale binding result after the polling version advances", async () => { - vi.useFakeTimers(); - const host = document.createElement("div"); - document.body.append(host); - let resolveOld!: (value: unknown) => void; - let resolveFresh!: (value: unknown) => void; - const oldResult = new Promise((resolve) => { - resolveOld = resolve; - }); - const freshResult = new Promise((resolve) => { - resolveFresh = resolve; - }); - const request = vi.fn().mockReturnValueOnce(oldResult).mockReturnValueOnce(freshResult); - const client = { - request, - addEventListener: vi.fn(() => () => {}), - } as unknown as GatewayBrowserClient; - const state = getWorkspaceState(host); - state.loaded = true; - state.activeSlug = "main"; - state.workspace = { - schemaVersion: 1, - workspaceVersion: 1, - tabs: [ - { - slug: "main", - title: "Main", - hidden: false, - widgets: [ - { - id: "cost", - kind: "builtin:stat-card", - title: "Cost", - grid: { x: 0, y: 0, w: 4, h: 2 }, - collapsed: false, - bindings: { value: { source: "rpc", method: "usage.cost" } }, - }, - ], - }, - ], - widgetsRegistry: {}, - prefs: { tabOrder: ["main"] }, - }; - - try { - render(renderWorkspace({ host, client, connected: true }), host); - expect(request).toHaveBeenCalledOnce(); - await vi.advanceTimersByTimeAsync(60_000); - render(renderWorkspace({ host, client, connected: true }), host); - expect(request).toHaveBeenCalledTimes(2); - - resolveFresh(2); - await freshResult; - await waitForFast(() => { - render(renderWorkspace({ host, client, connected: true }), host); - expect(host.querySelector(".workspace-stat__value")?.textContent).toBe("2"); - }); - - resolveOld(1); - await oldResult; - await Promise.resolve(); - render(renderWorkspace({ host, client, connected: true }), host); - expect(host.querySelector(".workspace-stat__value")?.textContent).toBe("2"); - } finally { - stopWorkspace(host); - host.remove(); - vi.useRealTimers(); - } - }); - - it("refreshes agent status when the canonical session list advances", async () => { - const host = document.createElement("div"); - document.body.append(host); - const state = getWorkspaceState(host); - let active = false; - const request = vi.fn(async (method: string) => - method === "sessions.list" - ? { sessions: [{ key: "agent:main", hasActiveRun: active }] } - : { total: 42 }, - ); - const client = { - request, - addEventListener: vi.fn(() => () => undefined), - } as unknown as GatewayBrowserClient; - state.loaded = true; - state.activeSlug = "main"; - state.workspace = { - schemaVersion: 1, - workspaceVersion: 1, - tabs: [ - { - slug: "main", - title: "Main", - hidden: false, - widgets: [ - { - id: "status", - kind: "builtin:agent-status", - title: "Agent status", - grid: { x: 0, y: 0, w: 6, h: 3 }, - collapsed: false, - bindings: { value: { source: "rpc", method: "sessions.list" } }, - }, - { - id: "usage", - kind: "builtin:usage", - title: "Usage", - grid: { x: 6, y: 0, w: 6, h: 3 }, - collapsed: false, - bindings: { value: { source: "rpc", method: "usage.status" } }, - }, - ], - }, - ], - widgetsRegistry: {}, - prefs: { tabOrder: ["main"] }, - }; - - try { - render(renderWorkspace({ host, client, connected: true }), host); - await waitForFast(() => expect(request).toHaveBeenCalledTimes(2)); - await waitForFast(() => { - render(renderWorkspace({ host, client, connected: true }), host); - expect( - host.querySelector("[data-test-id='workspace-agent-status']")?.textContent, - ).toContain("Idle"); - }); - - active = true; - render(renderWorkspace({ host, client, connected: true, sessionListRevision: 1 }), host); - await waitForFast(() => { - expect(request.mock.calls.filter(([method]) => method === "sessions.list")).toHaveLength(2); - }); - await waitForFast(() => { - render(renderWorkspace({ host, client, connected: true, sessionListRevision: 1 }), host); - expect( - host.querySelector("[data-test-id='workspace-agent-status']")?.textContent, - ).toContain("Busy"); - }); - expect(request.mock.calls.filter(([method]) => method === "usage.status")).toHaveLength(1); - } finally { - stopWorkspace(host); - host.remove(); - } - }); - - it("reloads a custom-widget frame after workspace changes and reconnects", async () => { - const host = document.createElement("div"); - document.body.append(host); - const state = getWorkspaceState(host); - const workspace = (workspaceVersion: number) => ({ - schemaVersion: 1, - workspaceVersion, - tabs: [ - { - slug: "main", - title: "Main", - hidden: false, - widgets: [ - { - id: "custom", - kind: "custom:revenue-chart", - title: "Revenue", - grid: { x: 0, y: 0, w: 6, h: 4 }, - collapsed: false, - }, - ], - }, - ], - widgetsRegistry: { "revenue-chart": { status: "approved" as const } }, - prefs: { tabOrder: ["main"] }, - }); - const request = vi - .fn() - .mockResolvedValueOnce({ - frameToken: "1111111111111111111111111111111111111111111", - frameExpiresAt: Date.now() + 60 * 60 * 1000, - manifest: { entrypoint: "old.html", bindings: [], capabilities: ["prompt:send"] }, - }) - .mockResolvedValueOnce({ - frameToken: "2222222222222222222222222222222222222222222", - frameExpiresAt: Date.now() + 60 * 60 * 1000, - manifest: { entrypoint: "new.html", bindings: [], capabilities: [] }, - }) - .mockResolvedValueOnce({ - frameToken: "3333333333333333333333333333333333333333333", - frameExpiresAt: Date.now() + 60 * 60 * 1000, - manifest: { entrypoint: "new.html", bindings: [], capabilities: [] }, - }); - const client = { - request, - addEventListener: vi.fn(() => () => undefined), - } as unknown as GatewayBrowserClient; - state.loaded = true; - state.activeSlug = "main"; - try { - state.workspace = workspace(1); - render(renderWorkspace({ host, client, connected: true }), host); - await waitForFast(() => expect(request).toHaveBeenCalledTimes(1)); - await waitForFast(() => { - render(renderWorkspace({ host, client, connected: true }), host); - expect(host.querySelector("iframe")?.getAttribute("src")).toContain("old.html"); - }); - - state.workspace = workspace(2); - render(renderWorkspace({ host, client, connected: true }), host); - await waitForFast(() => expect(request).toHaveBeenCalledTimes(2)); - await waitForFast(() => { - render(renderWorkspace({ host, client, connected: true }), host); - expect(host.querySelector("iframe")?.getAttribute("src")).toContain("new.html"); - }); - - render(renderWorkspace({ host, client, connected: false }), host); - render(renderWorkspace({ host, client, connected: true }), host); - await waitForFast(() => expect(request).toHaveBeenCalledTimes(3)); - await waitForFast(() => { - render(renderWorkspace({ host, client, connected: true }), host); - expect(host.querySelector("iframe")?.getAttribute("src")).toContain( - "3333333333333333333333333333333333333333333", - ); - }); - } finally { - stopWorkspace(host); - host.remove(); - } - }); -}); - -describe("drag ghost (#4)", () => { - afterEach(() => { - window.history.replaceState({}, "", "/"); - }); - - it("renders a snapped drop-target ghost while a drag is in flight", () => { - const host = document.createElement("div"); - document.body.append(host); - const client = { - request: vi.fn(async () => ({})), - addEventListener: vi.fn(() => () => {}), - } as unknown as GatewayBrowserClient; - const state = getWorkspaceState(host); - state.loaded = true; - state.workspace = doc; - state.activeSlug = "main"; - try { - render(renderWorkspace({ host, client, connected: true }), host); - const grid = host.querySelector(".workspace-grid"); - Object.defineProperty(grid, "clientWidth", { value: 720, configurable: true }); - // No ghost before a drag begins. - expect(host.querySelector('[data-test-id="workspace-drag-ghost"]')).toBeNull(); - const bar = host.querySelector(".workspace-widget__bar"); - bar!.dispatchEvent( - new PointerEvent("pointerdown", { bubbles: true, button: 0, clientX: 10, clientY: 10 }), - ); - render(renderWorkspace({ host, client, connected: true }), host); - // The ghost is present during the drag. - expect(host.querySelector('[data-test-id="workspace-drag-ghost"]')).not.toBeNull(); - window.dispatchEvent(new PointerEvent("pointerup", { clientX: 10, clientY: 10 })); - render(renderWorkspace({ host, client, connected: true }), host); - // Ghost gone once the drag settles. - expect(host.querySelector('[data-test-id="workspace-drag-ghost"]')).toBeNull(); - } finally { - stopWorkspace(host); - host.remove(); - } - }); -}); - -describe("mid-drag tab-switch cancellation", () => { - afterEach(() => { - window.history.replaceState({}, "", "/"); - }); - - it("cancels an in-flight drag on stopWorkspace so a later pointerup is a no-op", () => { - // The host IS the render container so gridMetrics/pointer targets resolve. - const host = document.createElement("div"); - document.body.append(host); - const request = vi.fn(async (..._args: unknown[]) => ({})); - const client = { - request, - addEventListener: vi.fn(() => () => {}), - } as unknown as GatewayBrowserClient; - const state = getWorkspaceState(host); - state.loaded = true; - state.workspace = doc; - state.activeSlug = "main"; - render(renderWorkspace({ host, client, connected: true }), host); - - // Grid clientWidth is 0 in jsdom; stub a real width so the drag begins. - const grid = host.querySelector(".workspace-grid"); - expect(grid).not.toBeNull(); - Object.defineProperty(grid, "clientWidth", { value: 720, configurable: true }); - - // Track window pointer listeners added during the drag. - const added = new Set(); - const originalAdd = window.addEventListener.bind(window); - const originalRemove = window.removeEventListener.bind(window); - const addSpy = vi - .spyOn(window, "addEventListener") - .mockImplementation((type: string, ...rest: unknown[]) => { - if (type === "pointermove" || type === "pointerup") { - added.add(type); - } - return (originalAdd as (t: string, ...r: unknown[]) => void)(type, ...rest); - }); - const removeSpy = vi - .spyOn(window, "removeEventListener") - .mockImplementation((type: string, ...rest: unknown[]) => { - if (type === "pointermove" || type === "pointerup") { - added.delete(type); - } - return (originalRemove as (t: string, ...r: unknown[]) => void)(type, ...rest); - }); - - try { - const bar = host.querySelector(".workspace-widget__bar"); - expect(bar).not.toBeNull(); - bar!.dispatchEvent( - new PointerEvent("pointerdown", { bubbles: true, button: 0, clientX: 10, clientY: 10 }), - ); - // The drag registered its window listeners. - expect(added.has("pointermove")).toBe(true); - expect(added.has("pointerup")).toBe(true); - - // Operator switches tabs mid-drag → the bundled view's stop hook fires. - stopWorkspace(host); - - // Listeners are gone… - expect(added.has("pointermove")).toBe(false); - expect(added.has("pointerup")).toBe(false); - - // …and a late pointerup does not resolve a move against the stale tab/client. - window.dispatchEvent(new PointerEvent("pointerup", { clientX: 400, clientY: 200 })); - expect(request.mock.calls.some(([method]) => method === "workspaces.widget.move")).toBe( - false, - ); - } finally { - addSpy.mockRestore(); - removeSpy.mockRestore(); - host.remove(); - } - }); -}); diff --git a/ui/src/pages/plugin/workspace-view.ts b/ui/src/pages/plugin/workspace-view.ts deleted file mode 100644 index e4addbe7eae..00000000000 --- a/ui/src/pages/plugin/workspace-view.ts +++ /dev/null @@ -1,1080 +0,0 @@ -// Control UI view renders the Workspaces bundled tab: tab strip, widget grid with -// hand-rolled pointer drag/drop + resize, empty states. Pure render fns — the -// controller owns lifecycle and `lib/workspace` owns data logic. - -import { html, nothing, type TemplateResult } from "lit"; -import type { GatewayBrowserClient } from "../../api/gateway.ts"; -import "../../components/modal-dialog.ts"; -import { icons } from "../../components/icons.ts"; -import "../../components/web-awesome.ts"; -import "../../components/web-awesome-tabs.ts"; -import { - loadWidgetManifestView, - type CustomWidgetHostContext, -} from "../../components/workspace-custom-widget.ts"; -import { - renderWorkspaceWidgetBody, - renderWidgetCell, - type WorkspaceCustomWidgetContext, - type WorkspaceWidgetCellCallbacks, -} from "../../components/workspace-widget-cell.ts"; -import { t } from "../../i18n/index.ts"; -import { - beginDrag, - collides, - WORKSPACE_GRID_GAP, - WORKSPACE_ROW_HEIGHT, - gridPlacementStyle, - gridRowCount, - nudgeRect, - resolveDrop, - updateDrag, - type WorkspaceDragState, -} from "../../lib/workspace/grid.ts"; -import { - approveWidget, - cancelWorkspaceLoadIntent, - clearActiveDrag, - customWidgetName, - customWidgetStatus, - findTab, - getWorkspaceState, - hiddenTabs, - hideWidget, - loadWorkspace, - moveWidget, - moveWidgetToTab, - orderedTabs, - removeWidgetFromTab, - registerActiveDrag, - resolveBinding, - setActiveWorkspaceSlug, - setWidgetCollapsed, - startBindingPolling, - subscribeToWorkspaceEvents, - updateWidgetTitle, - visibleTabs, - type WorkspaceBindingResult, - type WorkspaceUiState, -} from "../../lib/workspace/index.ts"; -import type { - WorkspaceBinding, - WorkspaceTab, - WorkspaceWidget, - WorkspaceDocument, - WidgetManifestView, -} from "../../lib/workspace/types.ts"; -import { buildCustomWidgetApprovalsSource } from "../../lib/workspace/widgets/custom-widget-approvals.ts"; -import type { BuiltinWidgetContext } from "../../lib/workspace/widgets/index.ts"; -import type { PreviewViewport } from "../../lib/workspace/widgets/types.ts"; -import { getSafeLocalStorage } from "../../local-storage.ts"; -import "../../styles/workspace.css"; -import { pluginTabRefFromSearch } from "./route.ts"; - -type WorkspaceProps = { - host: object; - client: GatewayBrowserClient | null; - connected: boolean; - /** Control UI embed policy for iframe-based builtins (defaults to strict). */ - embed?: BuiltinWidgetContext["embed"]; - onRequestUpdate: () => void; - /** Gateway HTTP base path for custom-widget iframe sources (L5). */ - basePath?: string; - /** Session key for custom-widget prompt dispatch (L5). */ - sessionKey?: string; - /** Advances when the shared session capability publishes a canonical list. */ - sessionListRevision?: number; - /** Controls custom-widget decisions; reads remain available without this scope. */ - canApproveWidgets?: boolean; -}; - -const DEFAULT_EMBED_CONTEXT: BuiltinWidgetContext["embed"] = { - embedSandboxMode: "strict", - allowExternalEmbedUrls: false, -}; - -// Per-host transient view state (menu, live drag) kept outside the data model so a -// broadcast refetch never clobbers an open menu or an in-flight drag. -type WorkspaceViewState = { - openMenuWidgetId: string | null; - drag: WorkspaceDragState | null; - /** Per-host preview viewport choices; intentionally not persisted in workspace data. */ - previewViewports: Map; - /** Last shared sessions.list revision folded into the binding cache. */ - sessionListRevision: number | undefined; - sessionListRevisionSeen: boolean; - sessionBindingGeneration: number; - /** Resolved binding cache keyed by widgetId; refreshed when the doc changes. */ - bindingResults: Map; - bindingLoads: Set; - bindingVersion: number; - /** Loaded custom-widget manifests keyed by widget name for one workspace version. */ - manifestCache: Map; - manifestLoads: Set; - manifestVersion: number; - manifestConnected: boolean; - manifestEpoch: number; - /** - * Monotonic data-refresh counter bumped by the per-widget polling timer. - * Folded into the binding cache key so a poll tick re-resolves data-widget - * bindings without a workspace-version change. - */ - dataVersion: number; - /** Active themed dialog (#12) for edit-title / move-to-tab, or null. */ - dialog: WorkspaceDialogState | null; - /** First-visit onboarding banner dismissed this session (#5); mirrors localStorage. */ - onboardingDismissed: boolean; -}; - -/** localStorage flag so the first-visit onboarding banner (#5) stays dismissed across reloads. */ -const ONBOARDING_DISMISS_KEY = "openclaw:control-ui:workspace-onboarding-dismissed:v1"; - -function isOnboardingDismissed(): boolean { - try { - return getSafeLocalStorage()?.getItem(ONBOARDING_DISMISS_KEY) === "1"; - } catch { - return false; - } -} - -function persistOnboardingDismissed(): void { - try { - getSafeLocalStorage()?.setItem(ONBOARDING_DISMISS_KEY, "1"); - } catch { - // Best effort — dismissing the hint is not a product failure. - } -} - -/** Themed-dialog state replacing the old window.prompt() flows (#12). */ -type WorkspaceDialogState = - | { kind: "editTitle"; slug: string; widgetId: string; title: string } - | { kind: "moveToTab"; slug: string; widgetId: string }; - -const workspaceViewStates = new WeakMap(); - -// Per-host document dismiss listener for the open kebab menu (#3). Installed while -// a menu is open so an outside pointerdown or Escape closes it; removed when the -// menu closes or the view stops. The details-based hidden-tabs menu closes via its -// own native outside-click/Escape once we drop `open` on the same signals. -type MenuDismissBinding = { - onPointerDown: (event: PointerEvent) => void; - onKeyDown: (event: KeyboardEvent) => void; -}; -const workspaceMenuDismiss = new WeakMap(); - -/** Remove the active menu-dismiss document listeners for `host`, if any. */ -function teardownMenuDismiss(host: object): void { - const binding = workspaceMenuDismiss.get(host); - if (!binding) { - return; - } - document.removeEventListener("pointerdown", binding.onPointerDown, true); - document.removeEventListener("keydown", binding.onKeyDown, true); - workspaceMenuDismiss.delete(host); -} - -/** - * Ensure the document-level dismiss listeners match whether a kebab menu is open. - * When open, an outside pointerdown or Escape clears `openMenuWidgetId`; a click - * inside the open menu/toggle is ignored so menu items still fire. - */ -function syncMenuDismiss( - host: object, - viewState: WorkspaceViewState, - requestUpdate: () => void, -): void { - const menuOpen = viewState.openMenuWidgetId !== null; - const active = workspaceMenuDismiss.has(host); - if (menuOpen === active) { - return; - } - if (!menuOpen) { - teardownMenuDismiss(host); - return; - } - const close = () => { - if (viewState.openMenuWidgetId === null) { - return; - } - viewState.openMenuWidgetId = null; - teardownMenuDismiss(host); - requestUpdate(); - }; - const onPointerDown = (event: PointerEvent) => { - const target = event.target; - if ( - target instanceof Element && - target.closest(".workspace-widget__menu, .workspace-widget__menu-toggle") - ) { - return; - } - close(); - }; - const onKeyDown = (event: KeyboardEvent) => { - if (event.key === "Escape") { - event.preventDefault(); - close(); - } - }; - document.addEventListener("pointerdown", onPointerDown, true); - document.addEventListener("keydown", onKeyDown, true); - workspaceMenuDismiss.set(host, { onPointerDown, onKeyDown }); -} - -/** View-level teardown: drop any menu-dismiss listeners. Called from the controller's stop. */ -export function stopWorkspaceView(host: object): void { - teardownMenuDismiss(host); -} - -function getViewState(host: object): WorkspaceViewState { - let state = workspaceViewStates.get(host); - if (!state) { - state = { - openMenuWidgetId: null, - drag: null, - previewViewports: new Map(), - sessionListRevision: undefined, - sessionListRevisionSeen: false, - sessionBindingGeneration: 0, - bindingResults: new Map(), - bindingLoads: new Set(), - bindingVersion: -1, - manifestCache: new Map(), - manifestLoads: new Set(), - manifestVersion: -1, - manifestConnected: false, - manifestEpoch: 0, - dataVersion: 0, - dialog: null, - onboardingDismissed: isOnboardingDismissed(), - }; - workspaceViewStates.set(host, state); - } - return state; -} - -/** Advance the data-refresh counter so the next render re-resolves bindings. */ -function bumpWorkspaceDataVersion(host: object): void { - getViewState(host).dataVersion += 1; -} - -/** The workspace tab slug requested via the `?ws=` deep-link query param. */ -function requestedWorkspaceSlug(search: string): string | null { - const params = new URLSearchParams(search); - const ws = params.get("ws")?.trim(); - return ws ? ws : null; -} - -/** Deep-link to a workspace tab: update `?ws=` and drive the router via popstate. */ -function navigateToWorkspaceTab(slug: string): void { - const url = new URL(window.location.href); - const ref = pluginTabRefFromSearch(url.search); - url.searchParams.set("plugin", ref.pluginId); - url.searchParams.set("id", ref.id); - url.searchParams.set("ws", slug); - window.history.pushState({}, "", `${url.pathname}${url.search}${url.hash}`); - window.dispatchEvent(new PopStateEvent("popstate")); -} - -/** Primary binding for a widget (first declared), if any. */ -function primaryBinding(widget: WorkspaceWidget): WorkspaceBinding | null { - const bindings = widget.bindings; - if (!bindings) { - return null; - } - const first = Object.values(bindings)[0]; - return first ?? null; -} - -function isCanonicalSessionBinding(binding: WorkspaceBinding | null): boolean { - return binding?.source === "rpc" && binding.method === "sessions.list"; -} - -function invalidateCanonicalSessionBindings( - viewState: WorkspaceViewState, - workspace: WorkspaceDocument | null, -): void { - for (const tab of workspace?.tabs ?? []) { - for (const widget of tab.widgets) { - if (!isCanonicalSessionBinding(primaryBinding(widget))) { - continue; - } - viewState.bindingResults.delete(widget.id); - viewState.bindingLoads.delete(widget.id); - } - } -} - -/** - * Cache key mixing the workspace version with the data-refresh counter: a doc - * change OR a poll tick both invalidate resolved bindings. Overflow-safe: only - * equality is compared. - */ -function bindingCacheKey(workspace: WorkspaceDocument, viewState: WorkspaceViewState): number { - return workspace.workspaceVersion * 1_000_003 + viewState.dataVersion; -} - -/** Kick off binding resolution for widgets on the active tab; cache per version. */ -function ensureBindings( - viewState: WorkspaceViewState, - client: GatewayBrowserClient | null, - workspace: WorkspaceDocument, - tab: WorkspaceTab, - requestUpdate: (() => void) | null, -): void { - const key = bindingCacheKey(workspace, viewState); - if (viewState.bindingVersion !== key) { - viewState.bindingResults.clear(); - viewState.bindingLoads.clear(); - viewState.bindingVersion = key; - } - for (const widget of tab.widgets) { - const binding = primaryBinding(widget); - if ( - !binding || - viewState.bindingResults.has(widget.id) || - viewState.bindingLoads.has(widget.id) - ) { - continue; - } - viewState.bindingLoads.add(widget.id); - const sessionBindingGeneration = isCanonicalSessionBinding(binding) - ? viewState.sessionBindingGeneration - : undefined; - void resolveBinding(client, binding).then((result) => { - if ( - viewState.bindingVersion !== key || - (sessionBindingGeneration !== undefined && - viewState.sessionBindingGeneration !== sessionBindingGeneration) - ) { - return; - } - viewState.bindingResults.set(widget.id, result); - viewState.bindingLoads.delete(widget.id); - requestUpdate?.(); - }); - } -} - -function gridMetrics(host: object): { width: number } { - const grid = - host instanceof HTMLElement ? host.querySelector(".workspace-grid") : null; - return { width: grid?.clientWidth ?? 0 }; -} - -/** - * First-visit onboarding banner (#5) teaching the two ways to add a tab: ask the - * agent (primary) or the CLI command (secondary). Dismissible; the flag persists - * in localStorage. The zero-tabs onboarding card is kept separately. - */ -function renderOnboardingBanner( - viewState: WorkspaceViewState, - requestUpdate: () => void, -): TemplateResult | typeof nothing { - if (viewState.onboardingDismissed) { - return nothing; - } - const dismiss = () => { - viewState.onboardingDismissed = true; - persistOnboardingDismissed(); - requestUpdate(); - }; - return html` -
- -
-
${t("workspaces.onboarding.title")}
-
${t("workspaces.onboarding.primary")}
-
- ${t("workspaces.onboarding.secondary")} - ${t("workspaces.empty.onboardingCommand")} -
-
- -
- `; -} - -function renderTabStrip(state: WorkspaceUiState, workspace: WorkspaceDocument): TemplateResult { - const tabs = visibleTabs(workspace); - const hidden = hiddenTabs(workspace); - return html` - ) => - navigateToWorkspaceTab(event.detail.name)} - > - ${tabs.map((tab) => { - return html` - - ${tab.icon && Object.hasOwn(icons, tab.icon) - ? html`` - : nothing} - ${tab.title} - - `; - })} - ${hidden.length > 0 - ? html` - ) => { - const slug = event.detail.item.value; - if (slug) { - navigateToWorkspaceTab(slug); - } - }} - > - - ${hidden.map( - (tab) => html` - - ${tab.title} - - `, - )} - - ` - : nothing} - - `; -} - -/** - * Load `widget.json` manifests for the APPROVED custom widgets on the active tab. - * Only approved widgets ever build an iframe, so only they need a manifest; a - * pending/rejected widget never fetches one. Workspace changes, reconnects, and - * approaching server expiry clear the cache: capabilities are approval-scoped - * and process-local, so they must never outlive either boundary. - */ -function ensureManifests( - viewState: WorkspaceViewState, - props: WorkspaceProps, - workspace: WorkspaceDocument, - tab: WorkspaceTab, -): void { - if (!props.connected || !props.client) { - return; - } - if (viewState.manifestVersion !== workspace.workspaceVersion) { - viewState.manifestCache.clear(); - viewState.manifestLoads.clear(); - viewState.manifestVersion = workspace.workspaceVersion; - } - const manifestVersion = viewState.manifestVersion; - const manifestEpoch = viewState.manifestEpoch; - for (const widget of tab.widgets) { - const name = customWidgetName(widget.kind); - const cached = name ? viewState.manifestCache.get(name) : undefined; - if (name && cached?.frameExpiresAt && cached.frameExpiresAt <= Date.now() + 60_000) { - viewState.manifestCache.delete(name); - } - if ( - !name || - customWidgetStatus(workspace, widget.kind) !== "approved" || - viewState.manifestCache.has(name) || - viewState.manifestLoads.has(name) - ) { - continue; - } - viewState.manifestLoads.add(name); - void loadWidgetManifestView(props.client, name).then((manifest) => { - // A fetch started for an older approval must not repopulate the cache after - // a workspace update invalidated it. - if ( - viewState.manifestVersion !== manifestVersion || - viewState.manifestEpoch !== manifestEpoch - ) { - return; - } - viewState.manifestLoads.delete(name); - if (manifest) { - viewState.manifestCache.set(name, manifest); - props.onRequestUpdate(); - } - }); - } -} - -/** Builds the L5 custom-widget context for one `custom:` widget, or null. */ -function buildCustomContext( - props: WorkspaceProps, - state: WorkspaceUiState, - viewState: WorkspaceViewState, - workspace: WorkspaceDocument, - widget: WorkspaceWidget, -): WorkspaceCustomWidgetContext | null { - const name = customWidgetName(widget.kind); - if (!name) { - return null; - } - const host: CustomWidgetHostContext = { - client: props.client, - basePath: props.basePath ?? "", - sessionKey: props.sessionKey ?? "main", - }; - const createdBy = workspace.widgetsRegistry[name]?.createdBy; - return { - status: customWidgetStatus(workspace, widget.kind), - ...(createdBy ? { createdBy } : {}), - manifest: viewState.manifestCache.get(name) ?? null, - host, - onApprove: () => void approveWidget(state, props.client, { name, decision: "approved" }), - onReject: () => void approveWidget(state, props.client, { name, decision: "rejected" }), - }; -} - -/** Shared ambient builtin state; full-bleed tabs must preserve every grid capability. */ -function buildBuiltinContext( - props: WorkspaceProps, - state: WorkspaceUiState, - viewState: WorkspaceViewState, - workspace: WorkspaceDocument, -): BuiltinWidgetContext { - const decideCustomWidget = props.canApproveWidgets - ? (name: string, decision: "approved" | "rejected") => { - void approveWidget(state, props.client, { name, decision }); - } - : undefined; - return { - basePath: props.basePath ?? "", - embed: props.embed ?? DEFAULT_EMBED_CONTEXT, - preview: { - getViewport: (widgetId, fallback) => viewState.previewViewports.get(widgetId) ?? fallback, - setViewport: (widgetId, viewport) => { - viewState.previewViewports.set(widgetId, viewport); - props.onRequestUpdate?.(); - }, - }, - customWidgetApprovals: buildCustomWidgetApprovalsSource( - workspace, - state.pendingApprovalNames, - decideCustomWidget, - ), - }; -} - -function renderGrid( - props: WorkspaceProps, - state: WorkspaceUiState, - viewState: WorkspaceViewState, - workspace: WorkspaceDocument, - tab: WorkspaceTab, -): TemplateResult { - ensureBindings(viewState, props.client, workspace, tab, props.onRequestUpdate ?? null); - ensureManifests(viewState, props, workspace, tab); - if (tab.widgets.length === 0) { - // #15: dashed placeholder card with an icon so an empty tab reads as an - // intentional drop zone. - return html` -
- -
${t("workspaces.empty.tabTitle")}
-
${t("workspaces.empty.tabSubtitle")}
-
- `; - } - if (tab.layout === "full") { - return renderFullBleed(props, state, viewState, workspace, tab); - } - const callbacks = makeCallbacks(props, state, viewState, tab); - const builtinContext = buildBuiltinContext(props, state, viewState, workspace); - const rows = gridRowCount(tab.widgets); - const minHeight = rows * WORKSPACE_ROW_HEIGHT + Math.max(0, rows - 1) * WORKSPACE_GRID_GAP; - return html` -
- ${tab.widgets.map((widget) => { - const custom = buildCustomContext(props, state, viewState, workspace, widget); - return renderWidgetCell({ - widget, - binding: viewState.bindingResults.get(widget.id) ?? null, - menuOpen: viewState.openMenuWidgetId === widget.id, - pending: state.pendingWidgetIds.has(widget.id), - dragging: viewState.drag?.widgetId === widget.id, - builtinContext, - callbacks, - ...(custom ? { custom } : {}), - }); - })} - ${renderDragGhost(viewState, tab)} -
- `; -} - -/** App-like tabs render their first widget through the same approval and sandbox path. */ -function renderFullBleed( - props: WorkspaceProps, - state: WorkspaceUiState, - viewState: WorkspaceViewState, - workspace: WorkspaceDocument, - tab: WorkspaceTab, -): TemplateResult { - const widget = tab.widgets[0]!; - const callbacks = makeCallbacks(props, state, viewState, tab); - const builtinContext = buildBuiltinContext(props, state, viewState, workspace); - const custom = buildCustomContext(props, state, viewState, workspace, widget); - return html`
- ${renderWorkspaceWidgetBody( - widget, - viewState.bindingResults.get(widget.id) ?? null, - builtinContext, - callbacks, - custom ?? undefined, - )} -
`; -} - -/** - * Snapped drop-target ghost for the active move/resize drag (#4). Placed in the - * same grid slot the drop would land in so the target is obvious. An overlapping - * (reject-bound) target reads distinctly via `--invalid`. - */ -function renderDragGhost( - viewState: WorkspaceViewState, - tab: WorkspaceTab, -): TemplateResult | typeof nothing { - const drag = viewState.drag; - if (!drag) { - return nothing; - } - const invalid = collides(drag.ghostRect, tab.widgets, drag.widgetId); - return html` - - `; -} - -function makeCallbacks( - props: WorkspaceProps, - state: WorkspaceUiState, - viewState: WorkspaceViewState, - tab: WorkspaceTab, -): WorkspaceWidgetCellCallbacks { - const requestUpdate = () => props.onRequestUpdate?.(); - const commitDrag = (widget: WorkspaceWidget, event: PointerEvent, mode: "move" | "resize") => { - const metrics = gridMetrics(props.host); - if (metrics.width <= 0) { - return; - } - const drag = beginDrag({ - widget, - mode, - clientX: event.clientX, - clientY: event.clientY, - metrics, - }); - viewState.drag = drag; - const target = event.target as Element; - if (target.setPointerCapture) { - target.setPointerCapture(event.pointerId); - } - // Once cancelled (tab-switch/disconnect via stopWorkspace), the window - // listeners are removed and any late pointerup becomes a no-op so it cannot - // fire moveWidget against a stale tab/client. - let settled = false; - const teardown = () => { - window.removeEventListener("pointermove", onMove); - window.removeEventListener("pointerup", onUp); - }; - const cancel = () => { - if (settled) { - return; - } - settled = true; - teardown(); - viewState.drag = null; - requestUpdate(); - }; - const onMove = (moveEvent: PointerEvent) => { - updateDrag(drag, moveEvent.clientX, moveEvent.clientY); - requestUpdate(); - }; - const onUp = () => { - if (settled) { - return; - } - settled = true; - teardown(); - clearActiveDrag(props.host); - const resolved = resolveDrop({ - requested: drag.ghostRect, - widgets: tab.widgets, - widgetId: widget.id, - }); - viewState.drag = null; - requestUpdate(); - if ( - resolved && - (resolved.x !== widget.grid.x || - resolved.y !== widget.grid.y || - resolved.w !== widget.grid.w || - resolved.h !== widget.grid.h) - ) { - void moveWidget(state, props.client, { - slug: tab.slug, - widgetId: widget.id, - grid: resolved, - }); - } - }; - window.addEventListener("pointermove", onMove); - window.addEventListener("pointerup", onUp); - registerActiveDrag(props.host, cancel); - }; - return { - onToggleCollapse: (widget) => - void setWidgetCollapsed(state, props.client, { - slug: tab.slug, - widgetId: widget.id, - collapsed: !widget.collapsed, - }), - onToggleMenu: (widget) => { - viewState.openMenuWidgetId = viewState.openMenuWidgetId === widget.id ? null : widget.id; - requestUpdate(); - }, - onCloseMenu: () => { - viewState.openMenuWidgetId = null; - requestUpdate(); - }, - onHide: (widget) => { - viewState.openMenuWidgetId = null; - // Hiding removes the widget from view and persists the hidden flag; distinct - // from remove, which deletes it from the document. - void hideWidget(state, props.client, { slug: tab.slug, widgetId: widget.id }); - }, - onRemove: (widget) => { - viewState.openMenuWidgetId = null; - viewState.previewViewports.delete(widget.id); - void removeWidgetFromTab(state, props.client, { slug: tab.slug, widgetId: widget.id }); - }, - onEditTitle: (widget) => { - viewState.openMenuWidgetId = null; - // #12: open the themed edit-title dialog instead of window.prompt(). - viewState.dialog = { - kind: "editTitle", - slug: tab.slug, - widgetId: widget.id, - title: widget.title, - }; - requestUpdate(); - }, - onMoveToTab: (widget) => { - viewState.openMenuWidgetId = null; - // #12: open the themed move-to-tab dialog (a select of existing tabs, not a - // free-text slug entry). - viewState.dialog = { kind: "moveToTab", slug: tab.slug, widgetId: widget.id }; - requestUpdate(); - }, - onMovePointerDown: (widget, event) => { - if (event.button !== 0) { - return; - } - event.preventDefault(); - commitDrag(widget, event, "move"); - }, - onResizePointerDown: (widget, event) => { - if (event.button !== 0) { - return; - } - event.preventDefault(); - event.stopPropagation(); - commitDrag(widget, event, "resize"); - }, - onKeyboardNudge: (widget, mode, direction) => { - const next = nudgeRect(widget.grid, mode, direction); - const resolved = resolveDrop({ requested: next, widgets: tab.widgets, widgetId: widget.id }); - if (resolved) { - void moveWidget(state, props.client, { - slug: tab.slug, - widgetId: widget.id, - grid: resolved, - }); - } - }, - }; -} - -/** - * Themed edit-title / move-to-tab dialog (#12), replacing window.prompt(). Reuses - * the app's openclaw-modal-dialog (Escape/backdrop cancel, focus trap) and the - * exec-approval card idiom. Move-to-tab offers a select of existing tabs. - */ -function renderDialog( - props: WorkspaceProps, - state: WorkspaceUiState, - viewState: WorkspaceViewState, -): TemplateResult | typeof nothing { - const dialog = viewState.dialog; - if (!dialog) { - return nothing; - } - const requestUpdate = () => props.onRequestUpdate?.(); - const close = () => { - viewState.dialog = null; - requestUpdate(); - }; - - if (dialog.kind === "editTitle") { - const title = t("workspaces.widget.editTitleTitle"); - const submit = (event: Event) => { - event.preventDefault(); - const input = (event.currentTarget as HTMLElement).querySelector( - "input[name='workspace-widget-title']", - ); - const next = input?.value.trim() ?? ""; - if (next && next !== dialog.title) { - void updateWidgetTitle(state, props.client, { - slug: dialog.slug, - widgetId: dialog.widgetId, - title: next, - }); - } - close(); - }; - return html` - -
-
-
${title}
-
- -
- - -
-
-
- `; - } - - const title = t("workspaces.widget.moveToTabTitle"); - const targets = state.workspace - ? orderedTabs(state.workspace).filter((candidate) => candidate.slug !== dialog.slug) - : []; - const submit = (event: Event) => { - event.preventDefault(); - const select = (event.currentTarget as HTMLElement).querySelector( - "select[name='workspace-move-target']", - ); - const toSlug = select?.value ?? ""; - if (toSlug && toSlug !== dialog.slug) { - void moveWidgetToTab(state, props.client, { - fromSlug: dialog.slug, - toSlug, - widgetId: dialog.widgetId, - }); - } - close(); - }; - return html` - -
-
-
${title}
-
- ${targets.length === 0 - ? html`
- ${t("workspaces.widget.moveToTabEmpty")} -
` - : html``} -
- - -
-
-
- `; -} - -export function renderWorkspace(props: WorkspaceProps): TemplateResult { - const state = getWorkspaceState(props.host); - const viewState = getViewState(props.host); - const sessionListRevision = props.sessionListRevision; - if (viewState.sessionListRevisionSeen && sessionListRevision !== viewState.sessionListRevision) { - // The shared session capability already coalesced the canonical event refresh. - // Drop only sessions.list values; unrelated data widgets keep their caches. - viewState.sessionBindingGeneration += 1; - invalidateCanonicalSessionBindings(viewState, state.workspace); - } - viewState.sessionListRevision = sessionListRevision; - viewState.sessionListRevisionSeen = true; - state.requestUpdate = props.onRequestUpdate ?? null; - // Keep the outside-click / Escape dismiss listeners in sync with the open kebab - // menu (#3). Cheap no-op when the open state is unchanged. - syncMenuDismiss(props.host, viewState, () => props.onRequestUpdate?.()); - - const requestedSlug = requestedWorkspaceSlug(window.location.search); - const active = props.connected; - // Gateway restart revokes every in-memory frame capability without changing - // workspaceVersion. A disconnect/reconnect therefore starts a new cache epoch. - if (viewState.manifestConnected !== active) { - viewState.manifestConnected = active; - viewState.manifestEpoch += 1; - viewState.manifestCache.clear(); - viewState.manifestLoads.clear(); - viewState.manifestVersion = -1; - } - subscribeToWorkspaceEvents(props.host, state, active ? props.client : null); - // Per-widget data refresh: a visibility-gated timer bumps the data version so - // the next render re-resolves data-widget bindings. stopWorkspace clears it on - // tab-leave/disconnect (logbook's stop discipline — no orphan timers). - startBindingPolling(props.host, active ? props.client : null, () => { - bumpWorkspaceDataVersion(props.host); - props.onRequestUpdate?.(); - }); - if (active && !state.loaded && !state.loading && !state.error) { - void loadWorkspace(state, props.client, { requestedSlug }); - } - - // Deep-link: a changed `?ws=` re-points the active tab without a refetch. - if (!requestedSlug) { - cancelWorkspaceLoadIntent(state); - } else if (state.workspace) { - setActiveWorkspaceSlug(state, state.workspace, requestedSlug); - } - - return html` -
- ${state.actionError - ? html`` - : nothing} - ${renderBody(props, state, viewState)} ${renderDialog(props, state, viewState)} -
- `; -} - -function renderBody( - props: WorkspaceProps, - state: WorkspaceUiState, - viewState: WorkspaceViewState, -): TemplateResult { - if (state.error) { - return html` - - `; - } - const workspace = state.workspace; - if (!workspace) { - // #19: skeleton cards instead of a bare "Loading…" line. - return html` -
- ${[0, 1, 2, 3, 4, 5].map(() => html`
`)} -
- `; - } - if (workspace.tabs.length === 0) { - return html` -
-
${t("workspaces.empty.onboardingTitle")}
-
${t("workspaces.empty.onboardingSubtitle")}
- ${t("workspaces.empty.onboardingCommand")} -
- `; - } - const tab = findTab(workspace, state.activeSlug) ?? visibleTabs(workspace)[0]; - if (!tab) { - return html`
-
${t("workspaces.empty.noVisibleTabs")}
-
`; - } - return html` - ${renderWorkspacesHeader(tab)} - ${renderOnboardingBanner(viewState, () => props.onRequestUpdate?.())} - ${renderTabStrip(state, workspace)} - - ${renderGrid(props, state, viewState, workspace, tab)} - - `; -} - -/** - * Page-header treatment for the Workspaces view (#7): the active workspace tab as - * the title, matching the app's .page-title header pattern - * idiom used by the other top-level pages. - */ -function renderWorkspacesHeader(tab: WorkspaceTab): TemplateResult { - return html` -
-
${tab.title}
-
- `; -} -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/ui/src/styles/workspace.css b/ui/src/styles/workspace.css deleted file mode 100644 index a7ddfd893f4..00000000000 --- a/ui/src/styles/workspace.css +++ /dev/null @@ -1,1043 +0,0 @@ -/* Workspaces bundled tab view (L3): tab strip, 12-column widget grid, cell chrome, - hand-rolled drag/resize affordances, provenance chip, error boundary. Uses the - shared Control UI theme tokens so agent-authored widgets and builtins match the - active palette (including the warm light theme). */ - -.workspace { - display: flex; - flex-direction: column; - gap: 12px; - flex: 1; - min-height: 0; - padding-bottom: 16px; - --workspace-row-height: 56px; - --workspace-grid-gap: 12px; -} - -.workspace__toast { - margin: 0; -} - -/* #7: page-header treatment for the Workspaces view (title = active tab). */ -.workspace-page-header { - display: flex; - flex-direction: column; - gap: 2px; -} - -/* --- Tab strip ----------------------------------------------------------- */ - -.workspace-tabs { - --track-width: 0; - --indicator-color: var(--accent); - display: block; - border-bottom: 1px solid var(--border); - padding-bottom: 8px; -} - -.workspace-tabs::part(tabs) { - display: flex; - align-items: center; - gap: 6px; - flex-wrap: wrap; -} - -.workspace-tabs::part(body) { - display: none; -} - -.workspace-tab::part(base), -button.workspace-tab { - display: inline-flex; - align-items: center; - gap: 6px; - height: 32px; - padding: 0 12px; - border: 1px solid transparent; - border-radius: var(--radius-md); - background: transparent; - color: var(--text-muted, var(--text)); - font: inherit; - cursor: pointer; - transition: - background 120ms var(--ease-out), - color 120ms var(--ease-out); -} - -.workspace-tab:hover::part(base), -button.workspace-tab:hover { - background: var(--bg-hover); - color: var(--text); -} - -.workspace-tab[active]::part(base) { - background: color-mix(in srgb, var(--accent) 16%, transparent); - border-color: color-mix(in srgb, var(--accent) 40%, var(--border)); - color: var(--text); -} - -.workspace-tab__icon svg { - width: 15px; - height: 15px; -} - -.workspace-tab__label { - white-space: nowrap; -} - -.workspace-tabs__hidden { - position: relative; - margin-inline-start: auto; -} - -.workspace-tab--overflow { - color: var(--text-muted, var(--text)); - /* #6 nit: keep the "Hidden (N)" affordance a compact single-line chip — the - icon + count must never wrap to a second line. */ - flex-wrap: nowrap; - white-space: nowrap; -} - -.workspace-tab--overflow .workspace-tab__icon svg { - width: 15px; - height: 15px; -} - -.workspace-tabs__hidden::part(menu) { - min-width: 160px; - padding: 4px; - background: var(--card); - border: 1px solid var(--border); - border-radius: var(--radius-md); - box-shadow: var(--shadow-md); -} - -.workspace-tabs__hidden-item { - color: var(--text); - font: inherit; -} - -/* --- Grid ---------------------------------------------------------------- */ - -.workspace-grid { - display: grid; - grid-template-columns: repeat(12, minmax(0, 1fr)); - grid-auto-rows: var(--workspace-row-height); - gap: var(--workspace-grid-gap); - align-content: start; - flex: 1; - min-height: 0; -} - -.workspace-fullbleed { - flex: 1; - min-width: 0; - min-height: 0; - overflow: auto; -} - -.workspace-fullbleed > .workspace-widget__custom, -.workspace-fullbleed > .workspace-embed__frame { - width: 100%; - height: 100%; - min-height: 480px; -} - -/* --- Widget cell --------------------------------------------------------- */ - -.workspace-widget { - display: flex; - flex-direction: column; - min-width: 0; - min-height: 0; - position: relative; - background: var(--card); - border: 1px solid var(--border); - /* #14: match .card's larger corner radius. Note the kebab menu and resize - dog-ear are portaled/absolute layers OUTSIDE this box, so keeping the card - itself clip-free is what lets the menu extend past the card (#1). */ - border-radius: var(--radius-lg); -} - -/* #16: a collapsed cell shrinks to its header height instead of reserving its - full grid slot — the body/resize handle are unmounted, so span a single row. */ -.workspace-widget--collapsed { - grid-row: auto !important; -} - -.workspace-widget--pending { - opacity: 0.7; -} - -/* #4: snapped drop-target ghost rendered in the grid during a move/resize drag. - Accent-tinted outline marks a valid target; --invalid reads distinctly (muted - red) for an overlap-rejected drop. */ -.workspace-ghost { - pointer-events: none; - border-radius: var(--radius-lg); - border: 2px dashed color-mix(in srgb, var(--accent) 70%, transparent); - background: color-mix(in srgb, var(--accent) 12%, transparent); - z-index: 5; -} - -.workspace-ghost--invalid { - border-color: color-mix(in srgb, var(--danger) 70%, transparent); - background: color-mix(in srgb, var(--danger) 12%, transparent); -} - -.workspace-widget--dragging { - outline: 2px dashed color-mix(in srgb, var(--accent) 60%, transparent); - outline-offset: 1px; - z-index: 10; -} - -.workspace-widget__bar { - display: flex; - align-items: center; - gap: 6px; - padding: 4px 6px 4px 4px; - border-bottom: 1px solid var(--border); - border-top-left-radius: var(--radius-lg); - border-top-right-radius: var(--radius-lg); - cursor: grab; - touch-action: none; - user-select: none; -} - -.workspace-widget__bar:active { - cursor: grabbing; -} - -.workspace-widget__collapse, -.workspace-widget__menu-toggle { - display: inline-flex; - align-items: center; - justify-content: center; - width: 24px; - height: 24px; - padding: 0; - border: none; - border-radius: var(--radius-sm); - background: transparent; - color: var(--text-muted, var(--text)); - cursor: pointer; -} - -.workspace-widget__collapse:hover, -.workspace-widget__menu-toggle:hover { - background: var(--bg-hover); - color: var(--text); -} - -/* #17: pressed/active cue while the kebab menu is open (aria-expanded=true). */ -.workspace-widget__menu-toggle[aria-expanded="true"] { - background: color-mix(in srgb, var(--accent) 16%, transparent); - color: var(--text); -} - -.workspace-widget__collapse svg, -.workspace-widget__menu-toggle svg { - width: 15px; - height: 15px; -} - -.workspace-widget__title { - flex: 1; - min-width: 0; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - /* #9: align to the app's .card-title token (15px / 600 / -0.02em). */ - font-size: 15px; - font-weight: 600; - letter-spacing: -0.02em; - color: var(--text-strong, var(--text)); -} - -.workspace-widget__provenance { - display: inline-flex; - align-items: center; - padding: 1px 6px; - border-radius: var(--radius-full, 999px); - background: color-mix(in srgb, var(--accent) 18%, transparent); - color: var(--accent); - font-size: 0.68em; - font-weight: 700; - letter-spacing: 0.04em; -} - -.workspace-widget__handle { - display: inline-flex; - align-items: center; - justify-content: center; - width: 20px; - height: 20px; - color: var(--text-dim, var(--text-muted, var(--text))); - cursor: grab; -} - -.workspace-widget__handle svg { - width: 14px; - height: 14px; -} - -.workspace-widget__menu { - z-index: 30; -} - -.workspace-widget__menu::part(menu) { - min-width: 160px; - padding: 4px; - background: var(--card); - border: 1px solid var(--border); - border-radius: var(--radius-md); - box-shadow: var(--shadow-md); -} - -.workspace-widget__menu-item { - text-align: start; - color: var(--text); -} - -.workspace-widget__menu-item--danger { - color: var(--danger); -} - -.workspace-widget__body { - flex: 1; - min-height: 0; - /* #14: roomier body padding to match the .card idiom. */ - padding: 14px 16px; - overflow: auto; - /* Clip body content to the card's rounded bottom corners now that the cell - itself no longer sets overflow:hidden (that would clip the kebab menu, #1). */ - border-bottom-left-radius: var(--radius-lg); - border-bottom-right-radius: var(--radius-lg); - /* #10: ease the body in on expand instead of an instant unmount/mount. The - collapse direction is carried by the grid-row shrink (#16). */ - animation: workspace-widget-expand 160ms var(--ease-out) both; -} - -@keyframes workspace-widget-expand { - from { - opacity: 0; - transform: translateY(-4px); - } - to { - opacity: 1; - transform: none; - } -} - -@media (prefers-reduced-motion: reduce) { - .workspace-widget__body { - animation: none; - } -} - -.workspace-widget__resize { - position: absolute; - inset-inline-end: 0; - inset-block-end: 0; - width: 16px; - height: 16px; - cursor: nwse-resize; - touch-action: none; - background: linear-gradient( - 135deg, - transparent 50%, - color-mix(in srgb, var(--border-strong) 80%, transparent) 50% - ); - /* #2: reveal the resize dog-ear only on cell hover/focus-within (or while a - resize drag is active, see --dragging below). */ - opacity: 0; - transition: opacity 120ms var(--ease-out); -} - -.workspace-widget:hover .workspace-widget__resize, -.workspace-widget:focus-within .workspace-widget__resize, -.workspace-widget--dragging .workspace-widget__resize { - opacity: 1; -} - -@media (prefers-reduced-motion: reduce) { - .workspace-widget__resize { - transition: none; - } -} - -.workspace-widget__placeholder { - color: var(--text-muted, var(--text)); - font-size: 0.85em; -} - -.workspace-widget__error { - display: flex; - flex-direction: column; - align-items: flex-start; - gap: 8px; - padding: 8px; - border-radius: var(--radius-sm); - background: var(--danger-subtle, color-mix(in srgb, var(--danger) 12%, transparent)); - color: var(--text); -} - -.workspace-widget__error-title { - font-weight: 600; - color: var(--danger); -} - -.workspace-widget__error-humane { - font-size: 0.9em; - color: var(--text-muted, var(--text)); -} - -.workspace-widget__error-detail, -.workspace-error-detail { - font-size: 0.82em; - color: var(--text-muted, var(--text)); -} - -.workspace-widget__error-detail summary, -.workspace-error-detail summary { - cursor: pointer; - color: var(--text-muted, var(--text)); -} - -.workspace-widget__error-message { - margin-top: 4px; - font-size: 0.85em; - color: var(--text-muted, var(--text)); - overflow-wrap: anywhere; -} - -.workspace-error-detail__text { - margin-top: 4px; - overflow-wrap: anywhere; -} - -/* --- Custom widget host (L5) --------------------------------------------- */ - -.workspace-widget__custom { - display: flex; - min-height: 160px; - height: 100%; -} - -.workspace-widget__frame { - display: block; - flex: 1; - width: 100%; - min-height: 160px; - border: 0; - border-radius: var(--radius-sm); - background: var(--card); -} - -.workspace-widget__approval { - display: flex; - flex-direction: column; - align-items: flex-start; - gap: 8px; - padding: 8px; -} - -.workspace-widget__approval-title { - font-weight: 600; - color: var(--text); -} - -.workspace-widget__approval-sub { - font-size: 0.85em; - color: var(--text-muted, var(--muted)); -} - -.workspace-widget__approval-actions { - display: flex; - gap: 8px; -} - -/* --- Builtin bodies ------------------------------------------------------ */ - -.workspace-stat { - display: flex; - flex-direction: column; - gap: 4px; -} - -/* #9: align to the app's .stat-value / .stat-label tokens (components.css). */ -.workspace-stat__value { - font-size: 24px; - font-weight: 700; - letter-spacing: -0.03em; - line-height: 1.1; - color: var(--text); -} - -.workspace-stat__label { - font-size: 12px; - font-weight: 500; - text-transform: uppercase; - letter-spacing: 0.04em; - color: var(--text-muted, var(--muted, var(--text))); -} - -.workspace-markdown { - font-size: 0.9em; -} - -/* Data-widget lists (sessions, cron, instances). */ -.workspace-list { - list-style: none; - margin: 0; - padding: 0; - display: flex; - flex-direction: column; - gap: 2px; - font-size: 0.85em; -} - -.workspace-list__row { - display: flex; - align-items: center; - gap: 8px; - padding: 4px 2px; - min-width: 0; -} - -.workspace-list__row--disabled { - opacity: 0.55; -} - -.workspace-list__link { - display: flex; - align-items: center; - gap: 8px; - flex: 1; - min-width: 0; - color: inherit; - text-decoration: none; - border-radius: var(--radius-sm); - padding: 2px 4px; -} - -.workspace-list__link:hover { - background: var(--bg-hover); -} - -.workspace-list__label { - flex: 1; - min-width: 0; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - color: var(--text); -} - -.workspace-list__meta { - color: var(--text-muted, var(--text)); - font-size: 0.9em; - white-space: nowrap; -} - -.workspace-custom-widget-approvals__row { - display: grid; - grid-template-columns: max-content minmax(0, 1fr) auto; - align-items: center; -} - -.workspace-custom-widget-approvals__actions { - display: flex; - grid-column: 3; - grid-row: 1 / span 2; - gap: 6px; -} - -.workspace-custom-widget-approvals__requested-by, -.workspace-custom-widget-approvals__permission { - grid-column: 1 / 3; - grid-row: 2; - white-space: normal; -} - -.workspace-custom-widget-approvals__permission { - grid-column: 1 / -1; - grid-row: 3; -} - -/* Live-run / health status dot. */ -.workspace-dot { - flex: none; - width: 8px; - height: 8px; - border-radius: 50%; - background: var(--text-dim, var(--text-muted, var(--text))); -} - -.workspace-dot--live, -.workspace-dot--ok { - background: var(--success, #22c55e); -} - -.workspace-dot--live { - box-shadow: 0 0 0 3px color-mix(in srgb, var(--success, #22c55e) 30%, transparent); -} - -.workspace-dot--warn { - background: var(--warning, #f59e0b); -} - -/* Compact status badge (cron/activity last-run status). */ -.workspace-badge { - flex: none; - padding: 1px 6px; - border-radius: var(--radius-full, 999px); - font-size: 0.9em; - font-weight: 600; - text-transform: lowercase; -} - -.workspace-badge--ok { - background: color-mix(in srgb, var(--success, #22c55e) 18%, transparent); - color: var(--success, #16a34a); -} - -.workspace-badge--error { - background: color-mix(in srgb, var(--danger) 16%, transparent); - color: var(--danger); -} - -.workspace-badge--muted { - background: var(--bg-muted); - color: var(--text-muted, var(--text)); -} - -/* Chart widget: dependency-free responsive SVG visuals. */ -.workspace-chart { - display: flex; - width: 100%; - height: 100%; - min-height: 40px; -} - -.workspace-chart__svg { - display: block; - width: 100%; - height: 100%; -} - -.workspace-chart__line { - stroke: var(--accent); - stroke-width: 1.5; - stroke-linecap: round; - stroke-linejoin: round; - vector-effect: non-scaling-stroke; -} - -.workspace-chart__point { - fill: var(--accent); -} - -.workspace-chart__area { - fill: color-mix(in srgb, var(--accent) 18%, transparent); -} - -.workspace-chart__bars rect { - fill: var(--accent); -} - -.workspace-chart__gauge-track, -.workspace-chart__gauge-fill { - stroke-width: 3; - stroke-linecap: round; - vector-effect: non-scaling-stroke; -} - -.workspace-chart__gauge-track { - stroke: var(--bg-muted); -} - -.workspace-chart__gauge-fill { - stroke: var(--accent); -} - -.workspace-chart__gauge-needle { - stroke: var(--text); - stroke-width: 1; - vector-effect: non-scaling-stroke; -} - -/* Usage mini-summary. */ -.workspace-usage { - display: flex; - gap: 20px; -} - -.workspace-usage__value { - font-size: 1.3em; - font-weight: 700; - color: var(--text); -} - -.workspace-usage__label { - font-size: 0.78em; - color: var(--text-muted, var(--text)); -} - -/* Table widget. */ -.workspace-table { - font-size: 0.82em; -} - -.workspace-table__grid { - width: 100%; - border-collapse: collapse; -} - -.workspace-table__grid th, -.workspace-table__grid td { - text-align: start; - padding: 4px 8px; - border-bottom: 1px solid var(--border); - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - max-width: 220px; -} - -.workspace-table__grid th { - color: var(--text-muted, var(--text)); - font-weight: 600; -} - -.workspace-table__footer { - margin-top: 6px; - color: var(--text-muted, var(--text)); - font-size: 0.9em; -} - -/* Activity feed. */ -.workspace-feed { - list-style: none; - margin: 0; - padding: 0; - display: flex; - flex-direction: column; - gap: 6px; - font-size: 0.84em; -} - -.workspace-feed__head { - display: flex; - align-items: center; - gap: 8px; - min-width: 0; -} - -.workspace-feed__title { - flex: 1; - min-width: 0; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - font-weight: 600; - color: var(--text); -} - -.workspace-feed__time { - color: var(--text-muted, var(--text)); - white-space: nowrap; -} - -.workspace-feed__detail { - color: var(--text-muted, var(--text)); - overflow-wrap: anywhere; -} - -/* iframe-embed widget. */ -.workspace-embed__frame { - display: block; - width: 100%; - height: 100%; - min-height: 120px; - border: none; - border-radius: var(--radius-sm); - background: var(--bg-muted); -} - -.workspace-preview { - display: flex; - flex-direction: column; - gap: 8px; - width: 100%; - height: 100%; - min-height: 0; -} - -.workspace-preview__toolbar { - display: flex; - align-items: center; - justify-content: space-between; - gap: 8px; - flex-wrap: wrap; -} - -.workspace-preview__viewports { - display: inline-flex; - gap: 4px; -} - -.workspace-preview__viewport, -.workspace-preview__reload { - padding: 2px 8px; - font-size: var(--font-size-xs, 12px); - color: var(--text-muted, var(--text)); - background: var(--bg-muted); - border: 1px solid var(--border, var(--bg-muted)); - border-radius: var(--radius-sm); - cursor: pointer; -} - -.workspace-preview__viewport:hover, -.workspace-preview__reload:hover, -.workspace-preview__viewport[aria-pressed="true"] { - color: var(--text); - border-color: var(--accent); -} - -.workspace-preview__viewport:focus-visible, -.workspace-preview__reload:focus-visible { - outline: 2px solid var(--accent); - outline-offset: 2px; -} - -.workspace-preview__frame-wrap { - flex: 1 1 auto; - display: flex; - justify-content: center; - min-height: 120px; -} - -.workspace-preview__frame-wrap--tablet .workspace-preview__frame { - width: 768px; - max-width: 100%; -} - -.workspace-preview__frame-wrap--mobile .workspace-preview__frame { - width: 375px; - max-width: 100%; -} - -/* --- Responsive: single column below ~900px ------------------------------ */ - -@media (max-width: 900px) { - .workspace-grid { - /* Collapse the 12-column grid to a single column so every widget stacks - full-width and reads on a narrow viewport. Explicit placements are - ignored because widgets flow into the single implicit column. */ - grid-template-columns: minmax(0, 1fr); - grid-auto-rows: auto; - } - - .workspace-grid > .workspace-widget { - grid-column: 1 / -1 !important; - grid-row: auto !important; - min-height: 160px; - } -} - -/* --- Empty / onboarding states ------------------------------------------ */ - -.workspace-empty { - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - gap: 8px; - flex: 1; - min-height: 200px; - padding: 32px; - text-align: center; - color: var(--text-muted, var(--text)); -} - -.workspace-empty__title { - font-size: 1.05em; - font-weight: 600; - color: var(--text); -} - -.workspace-empty__cmd { - margin-top: 4px; - padding: 4px 8px; - border-radius: var(--radius-sm); - background: var(--bg-muted); - font-family: var(--mono, monospace); - font-size: 0.85em; -} - -/* #15: give the empty-tab state a dashed placeholder card with an icon so it - reads as an intentional drop zone, not a rendering gap. */ -.workspace-empty--tab { - gap: 10px; - border: 1px dashed var(--border-strong, var(--border)); - border-radius: var(--radius-lg); - background: color-mix(in srgb, var(--card) 60%, transparent); -} - -.workspace-empty__icon { - color: var(--text-dim, var(--text-muted, var(--text))); -} - -.workspace-empty__icon svg { - width: 28px; - height: 28px; -} - -/* #5: first-visit onboarding banner teaching the two ways to add a tab. Matches - the app's card/callout banner idiom (dismissible, localStorage-backed). */ -.workspace-onboarding { - display: flex; - align-items: flex-start; - gap: 12px; - padding: 14px 16px; - border: 1px solid color-mix(in srgb, var(--accent) 40%, var(--border)); - border-radius: var(--radius-lg); - background: color-mix(in srgb, var(--accent) 8%, var(--card)); -} - -.workspace-onboarding__icon { - flex: none; - color: var(--accent); - margin-top: 1px; -} - -.workspace-onboarding__icon svg { - width: 18px; - height: 18px; -} - -.workspace-onboarding__body { - flex: 1; - min-width: 0; - display: flex; - flex-direction: column; - gap: 4px; -} - -.workspace-onboarding__title { - font-weight: 600; - color: var(--text-strong, var(--text)); -} - -.workspace-onboarding__sub { - font-size: 0.9em; - color: var(--text-muted, var(--muted, var(--text))); -} - -.workspace-onboarding__cmd { - margin-top: 2px; - padding: 2px 6px; - border-radius: var(--radius-sm); - background: var(--bg-muted); - font-family: var(--mono, monospace); - font-size: 0.82em; -} - -.workspace-onboarding__dismiss { - flex: none; - display: inline-flex; - align-items: center; - justify-content: center; - width: 24px; - height: 24px; - padding: 0; - border: none; - border-radius: var(--radius-sm); - background: transparent; - color: var(--text-muted, var(--text)); - cursor: pointer; -} - -.workspace-onboarding__dismiss:hover { - background: var(--bg-hover); - color: var(--text); -} - -.workspace-onboarding__dismiss svg { - width: 15px; - height: 15px; -} - -/* #12: themed edit-title / move-to-tab dialog inputs (mirrors the .field idiom). */ -.workspace-dialog__input { - border: 1px solid var(--input, var(--border)); - background: var(--card); - border-radius: var(--radius-md); - padding: 8px 12px; - color: var(--text); - outline: none; - box-shadow: inset 0 1px 0 var(--card-highlight, transparent); - transition: - border-color var(--duration-fast, 120ms) ease, - box-shadow var(--duration-fast, 120ms) ease; -} - -.workspace-dialog__input:focus-visible { - border-color: var(--ring, var(--accent)); - box-shadow: var(--focus-ring, 0 0 0 2px color-mix(in srgb, var(--accent) 40%, transparent)); -} - -select.workspace-dialog__input { - appearance: none; - cursor: pointer; -} - -/* #19: skeleton loading cards instead of a bare "Loading…" line. */ -.workspace-skeleton { - display: grid; - grid-template-columns: repeat(12, minmax(0, 1fr)); - gap: var(--workspace-grid-gap); - flex: 1; - align-content: start; -} - -.workspace-skeleton__card { - grid-column: span 4; - min-height: 120px; - border: 1px solid var(--border); - border-radius: var(--radius-lg); - background: linear-gradient( - 100deg, - var(--card) 30%, - color-mix(in srgb, var(--text) 6%, var(--card)) 50%, - var(--card) 70% - ); - background-size: 200% 100%; - animation: workspace-skeleton-shimmer 1.4s ease-in-out infinite; -} - -@keyframes workspace-skeleton-shimmer { - from { - background-position: 200% 0; - } - to { - background-position: -200% 0; - } -} - -@media (max-width: 900px) { - .workspace-skeleton { - grid-template-columns: minmax(0, 1fr); - } - - .workspace-skeleton__card { - grid-column: 1 / -1; - } -} - -@media (prefers-reduced-motion: reduce) { - .workspace-skeleton__card { - animation: none; - } -} From e7fe47d63fbc02ad122d474a035f53ae0b46b6e5 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 10:58:22 +0100 Subject: [PATCH 16/76] fix(browser): allow one retry after transient failures (#110607) * fix(browser): soften transient retry guidance * style(browser): format retry hint test * chore: keep release note in PR body --- .../client-fetch.attach-only.e2e.test.ts | 2 + .../client-fetch.loopback-auth.test.ts | 253 +++++++++++++++++- .../browser/src/browser/client-fetch.ts | 113 ++++++-- 3 files changed, 334 insertions(+), 34 deletions(-) diff --git a/extensions/browser/src/browser/client-fetch.attach-only.e2e.test.ts b/extensions/browser/src/browser/client-fetch.attach-only.e2e.test.ts index 37360e0e3bf..e0aa3f91837 100644 --- a/extensions/browser/src/browser/client-fetch.attach-only.e2e.test.ts +++ b/extensions/browser/src/browser/client-fetch.attach-only.e2e.test.ts @@ -75,6 +75,8 @@ describe("browser client fetch attachOnly diagnostics", () => { const message = thrown instanceof Error ? thrown.message : String(thrown); expect(message).toContain("browser profile is external to OpenClaw"); expect(message).toContain("Restarting the OpenClaw gateway will not launch it"); + expect(message).toContain("Retry the browser tool once"); + expect(message).toContain("If the same error persists"); expect(message).not.toContain("Restart the OpenClaw gateway"); expect(message).not.toContain("Do NOT retry the browser tool"); } finally { diff --git a/extensions/browser/src/browser/client-fetch.loopback-auth.test.ts b/extensions/browser/src/browser/client-fetch.loopback-auth.test.ts index c08b0fb6618..ba61fe74877 100644 --- a/extensions/browser/src/browser/client-fetch.loopback-auth.test.ts +++ b/extensions/browser/src/browser/client-fetch.loopback-auth.test.ts @@ -226,11 +226,16 @@ describe("fetchBrowserJson loopback auth", () => { expect(headers.get("authorization")).toBeNull(); }); - it("preserves dispatcher timeout context without no-retry hint", async () => { + it("preserves dispatcher timeout context with retry-once hint", async () => { mocks.dispatch.mockRejectedValueOnce(new Error("Chrome CDP handshake timeout")); await expectThrownBrowserFetchError(() => fetchBrowserJson<{ ok: boolean }>("/tabs"), { - contains: ["Chrome CDP handshake timeout", "Restart the OpenClaw gateway"], + contains: [ + "Chrome CDP handshake timeout", + "Restart the OpenClaw gateway", + "Retry the browser tool once", + "If the same error persists", + ], omits: ["Can't reach the OpenClaw browser control service", "Do NOT retry the browser tool"], }); }); @@ -267,6 +272,8 @@ describe("fetchBrowserJson loopback auth", () => { "Chrome CDP handshake timeout", "browser profile is external to OpenClaw", "Restarting the OpenClaw gateway will not launch it", + "Retry the browser tool once", + "If the same error persists", ], omits: ["Restart the OpenClaw gateway", "Do NOT retry the browser tool"], }, @@ -319,6 +326,8 @@ describe("fetchBrowserJson loopback auth", () => { "timed out", "browser profile is external to OpenClaw", "Restarting the OpenClaw gateway will not launch it", + "Retry the browser tool once", + "If the same error persists", ], omits: ["Restart the OpenClaw gateway", "Do NOT retry the browser tool"], }, @@ -342,7 +351,12 @@ describe("fetchBrowserJson loopback auth", () => { await expectThrownBrowserFetchError( () => fetchBrowserJson<{ ok: boolean }>("/tabs?profile=openclaw"), { - contains: ["Chrome CDP handshake timeout", "Restart the OpenClaw gateway"], + contains: [ + "Chrome CDP handshake timeout", + "Restart the OpenClaw gateway", + "Retry the browser tool once", + "If the same error persists", + ], omits: ["browser profile is external to OpenClaw", "Do NOT retry the browser tool"], }, ); @@ -357,7 +371,12 @@ describe("fetchBrowserJson loopback auth", () => { await expectThrownBrowserFetchError( () => fetchBrowserJson<{ ok: boolean }>("/tabs?profile=manual"), { - contains: ["Chrome CDP handshake timeout", "Restart the OpenClaw gateway"], + contains: [ + "Chrome CDP handshake timeout", + "Restart the OpenClaw gateway", + "Retry the browser tool once", + "If the same error persists", + ], omits: ["browser profile is external to OpenClaw", "Do NOT retry the browser tool"], }, ); @@ -380,7 +399,12 @@ describe("fetchBrowserJson loopback auth", () => { await expectThrownBrowserFetchError( () => fetchBrowserJson<{ ok: boolean }>("/tabs?profile=missing"), { - contains: ["Chrome CDP handshake timeout", "Restart the OpenClaw gateway"], + contains: [ + "Chrome CDP handshake timeout", + "Restart the OpenClaw gateway", + "Retry the browser tool once", + "If the same error persists", + ], omits: ["browser profile is external to OpenClaw", "Do NOT retry the browser tool"], }, ); @@ -406,6 +430,8 @@ describe("fetchBrowserJson loopback auth", () => { "Chrome CDP handshake timeout", "browser profile is external to OpenClaw", "Restarting the OpenClaw gateway will not launch it", + "Retry the browser tool once", + "If the same error persists", ], omits: ["Restart the OpenClaw gateway", "Do NOT retry the browser tool"], }); @@ -449,6 +475,51 @@ describe("fetchBrowserJson loopback auth", () => { }); }); + it("keeps transient dispatcher connection resets retryable once", async () => { + mocks.dispatch.mockRejectedValueOnce(new Error("Chrome CDP connection reset")); + + await expectThrownBrowserFetchError(() => fetchBrowserJson<{ ok: boolean }>("/tabs"), { + contains: [ + "Chrome CDP connection reset", + "Retry the browser tool once", + "If the same error persists", + ], + omits: ["Do NOT retry the browser tool"], + }); + }); + + it("uses top-level reset codes to classify dispatcher failures as transient", async () => { + mocks.dispatch.mockRejectedValueOnce( + Object.assign(new Error("socket closed"), { code: "ECONNRESET" }), + ); + + await expectThrownBrowserFetchError(() => fetchBrowserJson<{ ok: boolean }>("/tabs"), { + contains: ["socket closed", "Retry the browser tool once", "If the same error persists"], + omits: ["Do NOT retry the browser tool"], + }); + }); + + it("keeps refusal causes non-retryable when the outer error mentions a timeout", async () => { + const refused = Object.assign(new Error("connect refused"), { code: "ECONNREFUSED" }); + mocks.dispatch.mockRejectedValueOnce( + new Error("browser request timed out", { cause: refused }), + ); + + await expectThrownBrowserFetchError(() => fetchBrowserJson<{ ok: boolean }>("/tabs"), { + contains: ["browser request timed out", "Do NOT retry the browser tool"], + omits: ["Retry the browser tool once"], + }); + }); + + it("keeps disabled browser control failures non-retryable", async () => { + mocks.dispatch.mockRejectedValueOnce(new Error("browser control disabled")); + + await expectThrownBrowserFetchError(() => fetchBrowserJson<{ ok: boolean }>("/tabs"), { + contains: ["browser control disabled", "Do NOT retry the browser tool"], + omits: ["Retry the browser tool once"], + }); + }); + it("preserves validated structured errors from dispatcher routes", async () => { mocks.dispatch.mockResolvedValueOnce({ status: 409, @@ -540,7 +611,115 @@ describe("fetchBrowserJson loopback auth", () => { () => fetchBrowserJson<{ ok: boolean }>("http://127.0.0.1:18888/"), { contains: ["internal error"], - omits: ["rate limit"], + omits: ["rate limit", "Retry the browser tool once", "Do NOT retry the browser tool"], + }, + ); + }); + + it("keeps transient HTTP error payloads retryable once", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(JSON.stringify({ error: "Chrome CDP handshake timeout" }), { + status: 504, + }), + ), + ); + + await expectThrownBrowserFetchError( + () => fetchBrowserJson<{ ok: boolean }>("http://127.0.0.1:18888/"), + { + contains: [ + "Chrome CDP handshake timeout", + "Retry the browser tool once", + "If the same error persists", + ], + omits: ["Do NOT retry the browser tool"], + }, + ); + }); + + it.each([408, 504])("uses HTTP %i to classify generic payloads as transient", async (status) => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("request failed", { status })), + ); + + await expectThrownBrowserFetchError( + () => fetchBrowserJson<{ ok: boolean }>("http://127.0.0.1:18888/"), + { + contains: ["request failed", "Retry the browser tool once", "If the same error persists"], + omits: ["Do NOT retry the browser tool"], + }, + ); + }); + + it("does not mark client validation errors transient from timeout wording alone", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(JSON.stringify({ error: "invalid timeout value" }), { + status: 400, + }), + ), + ); + + await expectThrownBrowserFetchError( + () => fetchBrowserJson<{ ok: boolean }>("http://127.0.0.1:18888/"), + { + contains: ["invalid timeout value"], + omits: ["Retry the browser tool once", "Do NOT retry the browser tool"], + }, + ); + }); + + it("keeps pre-annotated persistent payload hints mutually exclusive", async () => { + const persistentHint = + "Do NOT retry the browser tool — it will keep failing. Use an alternative approach or inform the user that the browser is currently unavailable."; + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(JSON.stringify({ error: `browser request timed out. ${persistentHint}` }), { + status: 504, + }), + ), + ); + + await expectThrownBrowserFetchError( + () => fetchBrowserJson<{ ok: boolean }>("http://127.0.0.1:18888/"), + { + contains: ["browser request timed out", persistentHint], + omits: ["Retry the browser tool once"], + }, + ); + }); + + it("keeps transient dispatcher error payloads retryable once", async () => { + mocks.dispatch.mockResolvedValueOnce({ + status: 500, + body: { error: "read ECONNRESET" }, + }); + + await expectThrownBrowserFetchError(() => fetchBrowserJson<{ ok: boolean }>("/tabs"), { + contains: ["read ECONNRESET", "Retry the browser tool once", "If the same error persists"], + omits: ["Do NOT retry the browser tool"], + }); + }); + + it("keeps authentication failures non-retryable", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("Unauthorized", { status: 401 })), + ); + + await expectThrownBrowserFetchError( + () => fetchBrowserJson<{ ok: boolean }>("http://127.0.0.1:18888/"), + { + contains: ["Unauthorized", "Do NOT retry the browser tool"], + omits: ["Retry the browser tool once"], }, ); }); @@ -557,7 +736,7 @@ describe("fetchBrowserJson loopback auth", () => { }); }); - it("keeps absolute URL failures wrapped as reachability errors", async () => { + it("keeps transient absolute URL failures retryable once", async () => { vi.stubGlobal( "fetch", vi.fn(async () => { @@ -570,13 +749,51 @@ describe("fetchBrowserJson loopback auth", () => { { contains: [ "Can't reach the OpenClaw browser control service", - "Do NOT retry the browser tool", + "Retry the browser tool once", + "If the same error persists", ], + omits: ["Do NOT retry the browser tool"], + }, + ); + }); + + it("uses nested reset causes to classify generic fetch failures as transient", async () => { + const reset = Object.assign(new Error("socket closed"), { code: "ECONNRESET" }); + vi.stubGlobal( + "fetch", + vi.fn(async () => { + throw new TypeError("fetch failed", { cause: reset }); + }), + ); + + await expectThrownBrowserFetchError( + () => fetchBrowserJson<{ ok: boolean }>("http://example.com/"), + { + contains: ["fetch failed", "Retry the browser tool once", "If the same error persists"], + omits: ["Do NOT retry the browser tool"], }, ); }); - it("omits no-retry hint for absolute HTTP timeout failures", async () => { + it("uses nested refusal causes to keep unavailable services non-retryable", async () => { + const refused = Object.assign(new Error("connect refused"), { code: "ECONNREFUSED" }); + vi.stubGlobal( + "fetch", + vi.fn(async () => { + throw new TypeError("fetch failed", { cause: refused }); + }), + ); + + await expectThrownBrowserFetchError( + () => fetchBrowserJson<{ ok: boolean }>("http://example.com/"), + { + contains: ["fetch failed", "Do NOT retry the browser tool"], + omits: ["Retry the browser tool once"], + }, + ); + }); + + it("uses retry-once hint for absolute HTTP timeout failures", async () => { vi.stubGlobal( "fetch", vi.fn(async () => { @@ -587,7 +804,11 @@ describe("fetchBrowserJson loopback auth", () => { await expectThrownBrowserFetchError( () => fetchBrowserJson<{ ok: boolean }>("http://example.com/", { timeoutMs: 1234 }), { - contains: ["timed out after 1234ms"], + contains: [ + "timed out after 1234ms", + "Retry the browser tool once", + "If the same error persists", + ], omits: ["Do NOT retry the browser tool"], }, ); @@ -604,7 +825,11 @@ describe("fetchBrowserJson loopback auth", () => { await expectThrownBrowserFetchError( () => fetchBrowserJson<{ ok: boolean }>("http://example.com/", { timeoutMs: Number.NaN }), { - contains: ["timed out after 5000ms"], + contains: [ + "timed out after 5000ms", + "Retry the browser tool once", + "If the same error persists", + ], omits: ["NaNms", "Do NOT retry the browser tool"], }, ); @@ -628,7 +853,11 @@ describe("fetchBrowserJson loopback auth", () => { timeoutMs: Number.MAX_SAFE_INTEGER, }), { - contains: [`timed out after ${MAX_TIMER_TIMEOUT_MS}ms`], + contains: [ + `timed out after ${MAX_TIMER_TIMEOUT_MS}ms`, + "Retry the browser tool once", + "If the same error persists", + ], omits: ["Do NOT retry the browser tool"], }, ); diff --git a/extensions/browser/src/browser/client-fetch.ts b/extensions/browser/src/browser/client-fetch.ts index 9281df4ca9a..7101f0c951d 100644 --- a/extensions/browser/src/browser/client-fetch.ts +++ b/extensions/browser/src/browser/client-fetch.ts @@ -5,6 +5,7 @@ * in-process dispatcher, adding loopback auth and operator-facing diagnostics. */ import { parseBrowserHttpUrl } from "openclaw/plugin-sdk/browser-config"; +import { extractErrorCode, formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime"; import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; @@ -45,8 +46,10 @@ function browserServiceErrorFromPayload( status?: number, ): BrowserServiceError { const parsed = parseBrowserErrorPayload(value); + const message = parsed?.error ?? fallback; + const modelHint = resolveBrowserServiceModelHint(message, status); return new BrowserServiceError( - parsed?.error ?? fallback, + modelHint ? appendBrowserToolModelHint(message, modelHint) : message, parsed && "reason" in parsed ? parsed : undefined, status, ); @@ -126,9 +129,19 @@ function withLoopbackBrowserAuth( }); } -const BROWSER_TOOL_MODEL_HINT = +const BROWSER_TOOL_PERSISTENT_MODEL_HINT = "Do NOT retry the browser tool — it will keep failing. " + "Use an alternative approach or inform the user that the browser is currently unavailable."; +const BROWSER_TOOL_TRANSIENT_MODEL_HINT = + "This may be a transient browser error. Retry the browser tool once. " + + "If the same error persists, use an alternative approach or inform the user that the browser is currently unavailable."; + +// Retry history already lives in the model transcript. Keep this classifier stateless so one +// session's transient failure cannot suppress browser retries in another session. +const BROWSER_TRANSIENT_NETWORK_ERROR_RE = + /\b(?:ECONNRESET|ECONNABORTED|ENETRESET|ETIMEDOUT|EPIPE|EHOSTUNREACH|ENETUNREACH|EAI_AGAIN|UND_ERR_(?:CONNECT_TIMEOUT|HEADERS_TIMEOUT|BODY_TIMEOUT|SOCKET))\b|fetch failed|network error|other side closed|socket (?:hang up|terminated)|connection (?:reset|aborted|timed out)/i; +const BROWSER_PERSISTENT_FAILURE_RE = + /\bECONNREFUSED\b|connection refused|browser control (?:is )?(?:disabled|not enabled)|invalid (?:auth|authentication|credentials|password|token)|authentication (?:failed|required)|unauthorized/i; const BROWSER_ERROR_BODY_LIMIT_BYTES = 16 * 1024; // `response/body` supports 5M characters; 32 MiB covers worst-case JSON escaping while staying bounded. @@ -185,38 +198,87 @@ function normalizeErrorMessage(err: unknown): string { return String(err); } -function appendBrowserToolModelHint(message: string): string { - if (message.includes(BROWSER_TOOL_MODEL_HINT)) { - return message; - } - return `${message} ${BROWSER_TOOL_MODEL_HINT}`; +function appendBrowserToolModelHint(message: string, hint: string): string { + const messageWithoutHints = message + .replaceAll(BROWSER_TOOL_PERSISTENT_MODEL_HINT, "") + .replaceAll(BROWSER_TOOL_TRANSIENT_MODEL_HINT, "") + .trim(); + return `${messageWithoutHints} ${hint}`; } -type BrowserFetchFailureKind = "timeout" | "aborted" | "persistent"; +type BrowserFetchFailureKind = "timeout" | "aborted" | "transient-network" | "persistent"; function resolveBrowserFetchTimeoutMs(timeoutMs: number | undefined): number { return resolveTimerTimeoutMs(timeoutMs, 5000); } function classifyBrowserFetchFailure(err: unknown): BrowserFetchFailureKind { - const msg = normalizeErrorMessage(err); - const msgLower = normalizeLowercaseStringOrEmpty(msg); + const directCode = extractErrorCode(err); + const formatted = formatErrorMessage(err); + const detail = directCode ? `${formatted} | ${directCode}` : formatted; + const detailLower = normalizeLowercaseStringOrEmpty(detail); const nameLower = err instanceof Error ? normalizeLowercaseStringOrEmpty(err.name) : ""; + if (nameLower === "aborterror") { + return "aborted"; + } + if (BROWSER_PERSISTENT_FAILURE_RE.test(detail)) { + return "persistent"; + } const looksLikeTimeout = - nameLower.includes("timeout") || msgLower.includes("timed out") || msgLower.includes("timeout"); + nameLower.includes("timeout") || + detailLower.includes("timed out") || + detailLower.includes("timeout"); if (looksLikeTimeout) { return "timeout"; } + if (BROWSER_TRANSIENT_NETWORK_ERROR_RE.test(detail)) { + return "transient-network"; + } const looksLikeAbort = - nameLower === "aborterror" || - msgLower.includes("aborterror") || - msgLower.includes("aborted") || - msgLower.includes("abort") || - msgLower.includes("cancelled") || - msgLower.includes("canceled"); + detailLower.includes("aborterror") || + detailLower.includes("aborted") || + detailLower.includes("abort") || + detailLower.includes("cancelled") || + detailLower.includes("canceled"); return looksLikeAbort ? "aborted" : "persistent"; } +function isPersistentBrowserServiceFailure(message: string, status: number | undefined): boolean { + return status === 401 || BROWSER_PERSISTENT_FAILURE_RE.test(message); +} + +function resolveBrowserServiceModelHint( + message: string, + status: number | undefined, +): string | undefined { + if (message.includes(BROWSER_TOOL_PERSISTENT_MODEL_HINT)) { + return BROWSER_TOOL_PERSISTENT_MODEL_HINT; + } + if (message.includes(BROWSER_TOOL_TRANSIENT_MODEL_HINT)) { + return BROWSER_TOOL_TRANSIENT_MODEL_HINT; + } + if (isPersistentBrowserServiceFailure(message, status)) { + return BROWSER_TOOL_PERSISTENT_MODEL_HINT; + } + if (status === 408 || status === 504) { + return BROWSER_TOOL_TRANSIENT_MODEL_HINT; + } + if (status === undefined || status < 500 || status > 599) { + return undefined; + } + const kind = classifyBrowserFetchFailure(new Error(message)); + return kind === "timeout" || kind === "transient-network" + ? BROWSER_TOOL_TRANSIENT_MODEL_HINT + : undefined; +} + +function resolveBrowserToolModelHint(kind: BrowserFetchFailureKind): string | undefined { + if (kind === "timeout" || kind === "transient-network") { + return BROWSER_TOOL_TRANSIENT_MODEL_HINT; + } + return kind === "persistent" ? BROWSER_TOOL_PERSISTENT_MODEL_HINT : undefined; +} + async function discardResponseBody(res: Response): Promise { try { await res.body?.cancel(); @@ -230,8 +292,8 @@ function enhanceDispatcherPathError(url: string, err: unknown): Error { const kind = classifyBrowserFetchFailure(err); const ownership = resolveDispatcherBrowserControlOwnership(url); const operatorHint = resolveBrowserFetchOperatorHint(url, { ownership }); - const suffix = - kind === "persistent" ? `${operatorHint} ${BROWSER_TOOL_MODEL_HINT}` : operatorHint; + const modelHint = resolveBrowserToolModelHint(kind); + const suffix = modelHint ? `${operatorHint} ${modelHint}` : operatorHint; const normalized = msg.endsWith(".") ? msg : `${msg}.`; return new Error(`${normalized} ${suffix}`, err instanceof Error ? { cause: err } : undefined); } @@ -242,7 +304,7 @@ function enhanceBrowserFetchError(url: string, err: unknown, timeoutMs: number): const kind = classifyBrowserFetchFailure(err); if (kind === "timeout") { return new Error( - `Can't reach the OpenClaw browser control service (timed out after ${timeoutMs}ms). ${operatorHint}`, + `Can't reach the OpenClaw browser control service (timed out after ${timeoutMs}ms). ${operatorHint} ${BROWSER_TOOL_TRANSIENT_MODEL_HINT}`, err instanceof Error ? { cause: err } : undefined, ); } @@ -252,9 +314,16 @@ function enhanceBrowserFetchError(url: string, err: unknown, timeoutMs: number): err instanceof Error ? { cause: err } : undefined, ); } + if (kind === "transient-network") { + return new Error( + `Can't reach the OpenClaw browser control service. ${operatorHint} (${msg}) ${BROWSER_TOOL_TRANSIENT_MODEL_HINT}`, + err instanceof Error ? { cause: err } : undefined, + ); + } return new Error( appendBrowserToolModelHint( `Can't reach the OpenClaw browser control service. ${operatorHint} (${msg})`, + BROWSER_TOOL_PERSISTENT_MODEL_HINT, ), err instanceof Error ? { cause: err } : undefined, ); @@ -298,7 +367,7 @@ async function fetchHttpJson( // Do not reflect upstream response text into the error surface (log/agent injection risk) await discardResponseBody(res); throw new BrowserServiceError( - `${resolveBrowserRateLimitMessage(url)} ${BROWSER_TOOL_MODEL_HINT}`, + `${resolveBrowserRateLimitMessage(url)} ${BROWSER_TOOL_PERSISTENT_MODEL_HINT}`, ); } // Overflow cancels the stream and releases its reader lock before the guarded fetch below. @@ -420,7 +489,7 @@ export async function fetchBrowserJson( if (isRateLimitStatus(result.status)) { // Do not reflect upstream response text into the error surface (log/agent injection risk) throw new BrowserServiceError( - `${resolveBrowserRateLimitMessage(url)} ${BROWSER_TOOL_MODEL_HINT}`, + `${resolveBrowserRateLimitMessage(url)} ${BROWSER_TOOL_PERSISTENT_MODEL_HINT}`, ); } throw browserServiceErrorFromPayload(result.body, `HTTP ${result.status}`, result.status); From 7a7c158210bc578e438c174a84cf93d7f3cbb50f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 11:01:08 +0100 Subject: [PATCH 17/76] perf(ci): parallelize gateway watch artifact check (#110609) --- .github/workflows/ci.yml | 63 ++++++++++++++----------- docs/ci.md | 2 +- test/scripts/ci-workflow-guards.test.ts | 15 ++++-- 3 files changed, 46 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b96b75211b..9b1eedd3607 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1137,6 +1137,7 @@ jobs: id: built_artifact_checks if: needs.preflight.outputs.run_checks == 'true' || needs.preflight.outputs.run_checks_node_core_dist == 'true' || needs.preflight.outputs.run_check_additional == 'true' env: + PARALLEL_GATEWAY_WATCH: ${{ runner.environment != 'github-hosted' && 'true' || 'false' }} RUN_CHANNELS: ${{ needs.preflight.outputs.run_checks }} RUN_CORE_SUPPORT_BOUNDARY: ${{ needs.preflight.outputs.run_checks_node_core_dist }} RUN_GATEWAY_WATCH: ${{ needs.preflight.outputs.run_check_additional }} @@ -1164,6 +1165,29 @@ jobs: pids+=("$!") } + wait_checks() { + local index name log pid result + for index in "${!pids[@]}"; do + name="${names[$index]}" + log="${logs[$index]}" + pid="${pids[$index]}" + + if wait "$pid"; then + result="success" + else + result="failure" + fi + + echo "::group::${name} log" + cat "$log" + echo "::endgroup::" + results["$name"]="$result" + done + names=() + pids=() + logs=() + } + if [ "$RUN_CHANNELS" = "true" ]; then start_check "channels" env \ NODE_OPTIONS=--max-old-space-size=8192 \ @@ -1178,36 +1202,19 @@ jobs: node scripts/run-vitest.mjs run --config test/vitest/vitest.full-core-support-boundary.config.ts fi - for index in "${!pids[@]}"; do - name="${names[$index]}" - log="${logs[$index]}" - pid="${pids[$index]}" - - if wait "$pid"; then - result="success" - else - result="failure" - fi - - echo "::group::${name} log" - cat "$log" - echo "::endgroup::" - results["$name"]="$result" - done + if [ "$RUN_GATEWAY_WATCH" = "true" ] && [ "$PARALLEL_GATEWAY_WATCH" = "true" ]; then + start_check "gateway-watch" \ + node scripts/check-gateway-watch-regression.mjs --skip-build + fi - if [ "$RUN_GATEWAY_WATCH" = "true" ]; then - log="${RUNNER_TEMP}/gateway-watch.log" - echo "starting gateway-watch: node scripts/check-gateway-watch-regression.mjs --skip-build" - if node scripts/check-gateway-watch-regression.mjs --skip-build >"$log" 2>&1; then - result="success" - else - result="failure" - fi + wait_checks - echo "::group::gateway-watch log" - cat "$log" - echo "::endgroup::" - results["gateway-watch"]="$result" + # Preserve the low-core hosted path; concurrent Vitest can otherwise + # starve the Gateway readiness deadline used by this regression gate. + if [ "$RUN_GATEWAY_WATCH" = "true" ] && [ "$PARALLEL_GATEWAY_WATCH" != "true" ]; then + start_check "gateway-watch" \ + node scripts/check-gateway-watch-regression.mjs --skip-build + wait_checks fi for name in channels core-support-boundary gateway-watch; do diff --git a/docs/ci.md b/docs/ci.md index 69a2e81aa12..61d22c79667 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -121,7 +121,7 @@ The slowest Node test families are split or balanced so each job stays small wit - The build-artifact job also persists content-fingerprinted `build-all` step outputs. CI's self-built plugin SDK declarations hash the complete repository-owned TypeScript/JSON source graph, exclude installed and generated directories, and restore both flat declarations and package bridges after `tsdown` clears `dist`. Documentation, workflow, plugin, and other changes outside that graph can reuse the declaration snapshot; source changes rebuild it before the export gate runs. - Full declaration builds split `tsdown` into AI, workspace-package, and unified groups. Each group caches declarations only, then still rebuilds runtime JavaScript before restoring those declarations. Core or plugin changes therefore invalidate only the large unified graph, while workspace-package changes conservatively invalidate every dependent declaration group. Public full builds generally use an immutable Actions cache; coarse restore keys seed partial changes, per-group content fingerprints reject stale data, and GitHub's cache quota evicts old generations. The weekly Node 22 lane instead publishes a 14-day artifact after successful `main` runs and restores only artifacts whose immutable producer identity resolves to that workflow on `main`, avoiding quota churn without allowing PR code to write a shared cache. Private-QA declarations are never persisted in Actions caches because cache namespaces are not confidentiality boundaries. - `check-additional-*` stripes the supplemental boundary guard list (`scripts/run-additional-boundary-checks.mjs`) into one prompt-heavy shard (`check-additional-boundaries-a`, which includes the Codex prompt snapshot drift check) and one combined shard for the remaining stripes (`check-additional-boundaries-bcd`), each running independent guards concurrently and printing per-check timings. Package-boundary compile/canary work stays together, and runtime topology architecture runs separately from the gateway watch coverage embedded in `build-artifacts`. -- Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built. +- On the 32-vCPU self-hosted build runner, Gateway watch, channel tests, and the core support-boundary shard start together inside `build-artifacts` after `dist/` and `dist-runtime/` are already built. GitHub-hosted fallback runs keep Gateway watch serial so low-core contention cannot consume its readiness deadline. Once admitted, canonical Linux CI permits up to 28 concurrent Node test jobs and 12 for the smaller fast/check lanes; Windows and Android stay at two because diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index bf27e4f1954..73d32bf1612 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -4087,7 +4087,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" expect(saveStep.with.key).toContain("dist-build-v2-"); }); - it("runs gateway watch after parallel built artifact checks", () => { + it("parallelizes gateway watch only on the large self-hosted build runner", () => { const workflow = readCiWorkflow(); const buildArtifactSteps = workflow.jobs["build-artifacts"].steps; const builtArtifactChecks = buildArtifactSteps.find( @@ -4095,15 +4095,20 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" ); const run = builtArtifactChecks.run; + expect(builtArtifactChecks.env.PARALLEL_GATEWAY_WATCH).toBe( + "${{ runner.environment != 'github-hosted' && 'true' || 'false' }}", + ); expect(run).toContain('start_check "channels"'); expect(run).toContain('start_check "core-support-boundary"'); - expect(run).not.toContain('start_check "gateway-watch"'); - expect(run.indexOf('for index in "${!pids[@]}"')).toBeLessThan( - run.indexOf('if [ "$RUN_GATEWAY_WATCH" = "true" ]; then'), + expect(run).toContain('start_check "gateway-watch"'); + expect(run).toContain( + 'if [ "$RUN_GATEWAY_WATCH" = "true" ] && [ "$PARALLEL_GATEWAY_WATCH" = "true" ]; then', ); expect(run).toContain( - 'node scripts/check-gateway-watch-regression.mjs --skip-build >"$log" 2>&1', + 'if [ "$RUN_GATEWAY_WATCH" = "true" ] && [ "$PARALLEL_GATEWAY_WATCH" != "true" ]; then', ); + expect(run).toContain("wait_checks()"); + expect(run.match(/wait_checks$/gmu)).toHaveLength(2); }); it("keeps docs i18n CI on the workflow-owned patched Go toolchain", () => { From 8c95ec95ca50532d01d02affd150b691b0f178f4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 11:02:38 +0100 Subject: [PATCH 18/76] fix(ui): show sidebar connection state only on sustained disconnect (#110567) --- ui/src/components/app-sidebar.ts | 60 +++++++++++++++---- ui/src/components/sidebar-agent-card.ts | 21 ++++--- ui/src/styles/layout.css | 28 ++++----- .../test-helpers/app-sidebar-cases/basics.ts | 41 ++++++++++++- ui/src/test-helpers/app-sidebar.ts | 1 + 5 files changed, 112 insertions(+), 39 deletions(-) diff --git a/ui/src/components/app-sidebar.ts b/ui/src/components/app-sidebar.ts index eeea929fbd1..d0c387b1911 100644 --- a/ui/src/components/app-sidebar.ts +++ b/ui/src/components/app-sidebar.ts @@ -1,4 +1,4 @@ -import { html, nothing } from "lit"; +import { html, nothing, type PropertyValues } from "lit"; import { state } from "lit/decorators.js"; import type { NavigationRouteId } from "../app-navigation.ts"; import { pathForRoute } from "../app-route-paths.ts"; @@ -34,9 +34,13 @@ import { const PALETTE_SHORTCUT = /Mac|iP(hone|ad|od)/i.test(globalThis.navigator?.platform ?? "") ? "⌘K" : "Ctrl K"; +const OFFLINE_INDICATOR_DELAY_MS = 2_000; class AppSidebar extends AppSidebarSessionListElement { @state() private logoVisit: LobsterLogoVisitDetail | null = null; + @state() private debouncedDisconnected = false; + + private offlineIndicatorTimer: ReturnType | null = null; constructor() { super(); @@ -44,6 +48,38 @@ class AppSidebar extends AppSidebarSessionListElement { this.addEventListener(LOBSTER_LOGO_VISIT_EVENT, this.handleLogoVisit as EventListener); } + override connectedCallback() { + super.connectedCallback(); + this.syncOfflineIndicator(); + } + + override disconnectedCallback() { + this.syncOfflineIndicator(false); + super.disconnectedCallback(); + } + + protected override willUpdate(changed: PropertyValues) { + if (changed.has("connected")) { + this.syncOfflineIndicator(); + } + } + + private syncOfflineIndicator(schedule = !this.connected) { + if (this.offlineIndicatorTimer !== null) { + globalThis.clearTimeout(this.offlineIndicatorTimer); + this.offlineIndicatorTimer = null; + } + this.debouncedDisconnected = false; + if (!schedule) { + return; + } + // Both sidebar signals share one grace window so brief transport blips stay quiet. + this.offlineIndicatorTimer = globalThis.setTimeout(() => { + this.offlineIndicatorTimer = null; + this.debouncedDisconnected = true; + }, OFFLINE_INDICATOR_DELAY_MS); + } + private readonly handleLogoVisit = (event: Event) => { const detail = (event as CustomEvent).detail; // A lookless visit is a logo scare: the brand mark hides (the img gets @@ -99,7 +135,7 @@ class AppSidebar extends AppSidebarSessionListElement { .agentName=${cardName} .avatarUrl=${cardAgent ? resolveAgentAvatarUrl(cardAgent) : null} .avatarText=${cardAvatarText} - .connected=${this.connected} + .offline=${this.debouncedDisconnected} .statusLabel=${gatewayStatus} .subtitle=${this.agentChipSubtitle(cardAgentId)} .menuOpen=${this.agentMenuPosition !== null} @@ -209,15 +245,17 @@ class AppSidebar extends AppSidebarSessionListElement { .gatewayVersion=${this.gatewayVersion} .onNavigate=${(routeId: "about") => this.onNavigate?.(routeId)} > - + ${this.debouncedDisconnected + ? html`${t( + "common.offline", + )}` + : nothing} + + ` + : nothing} ${abortAction} ` : props.canAbort @@ -1875,23 +1909,6 @@ function renderChatPrimaryActions(props: ChatRunControlsProps) { > - ${props.onToggleVideo - ? html` - - - - ` - : nothing} `} `; } @@ -2271,13 +2288,16 @@ export function renderChatComposer(props: ChatComposerProps) { voiceStatus: props.realtimeTalkStatus, voiceDetail: props.realtimeTalkDetail, voiceInputLevel: props.realtimeTalkInputLevel, + voiceVideoCapable: props.realtimeTalkVideoCapable, + voiceVideoEnabled: Boolean(props.realtimeTalkVideoStream), + voiceVideoPending: props.realtimeTalkVideoPending, onAbort: props.onAbort, onExport: () => exportMarkdown(props), onNewSession: props.onNewSession, onSend: handleSend, onStoreDraft: () => {}, onToggleVoice: props.onToggleRealtimeTalk ? handleVoicePrimaryAction : undefined, - onToggleVideo: props.onToggleRealtimeVideo, + onToggleCamera: props.onToggleRealtimeCamera, }; const slashMenuVisible = props.connected && canCompose && isSlashMenuVisible(state); const activeSlashMenuOptionId = getActiveSlashMenuOptionId(state, props.paneId); @@ -2367,9 +2387,11 @@ export function renderChatComposer(props: ChatComposerProps) { ${renderChatAttachmentInputs({ ...props, disabled: !canCompose })} ${renderChatVoiceError({ - status: props.realtimeTalkStatus, + status: props.realtimeTalkCameraError ? "error" : props.realtimeTalkStatus, detail: props.realtimeTalkDetail, - onDismissError: props.onDismissRealtimeTalkError, + onDismissError: props.realtimeTalkCameraError + ? undefined + : props.onDismissRealtimeTalkError, })} ${props.realtimeTalkVideoStream ? html` diff --git a/ui/src/pages/chat/realtime-talk-google-live-video.test.ts b/ui/src/pages/chat/realtime-talk-google-live-video.test.ts index 0794e6a8cf8..e28c33a50fe 100644 --- a/ui/src/pages/chat/realtime-talk-google-live-video.test.ts +++ b/ui/src/pages/chat/realtime-talk-google-live-video.test.ts @@ -79,7 +79,6 @@ function createTransport(callbacks: RealtimeTalkCallbacks) { callbacks, client: { request: vi.fn(), addEventListener: vi.fn() } as never, sessionKey: "main", - videoEnabled: true, }, ); } @@ -102,12 +101,12 @@ describe("Google Live Video Talk", () => { const audioStop = vi.fn(); const videoStop = vi.fn(); const audioTrack = { stop: audioStop } as unknown as MediaStreamTrack; - const videoTrack = { + const videoTrack = Object.assign(new EventTarget(), { stop: videoStop, readyState: "live", enabled: true, muted: false, - } as unknown as MediaStreamTrack; + }) as unknown as MediaStreamTrack; const audio = { getAudioTracks: () => [audioTrack], getTracks: () => [audioTrack], @@ -116,21 +115,8 @@ describe("Google Live Video Talk", () => { getVideoTracks: () => [videoTrack], getTracks: () => [videoTrack], } as unknown as MediaStream; - class TestMediaStream { - constructor(readonly tracks: MediaStreamTrack[]) {} - getAudioTracks() { - return [audioTrack]; - } - getVideoTracks() { - return [videoTrack]; - } - getTracks() { - return this.tracks; - } - } const getUserMedia = vi.fn().mockResolvedValueOnce(audio).mockResolvedValueOnce(camera); vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); - vi.stubGlobal("MediaStream", TestMediaStream); const originalCreateElement = document.createElement.bind(document); vi.spyOn(document, "createElement").mockImplementation((tagName: string) => { @@ -156,6 +142,9 @@ describe("Google Live Video Talk", () => { const transport = createTransport({ onStatus, onVideoStream }); await transport.start(); + expect(getUserMedia).toHaveBeenCalledOnce(); + expect(onVideoStream).not.toHaveBeenCalled(); + await transport.setVideoEnabled(true); const ws = FakeGoogleLiveWebSocket.instance; if (!ws) { throw new Error("missing Google Live WebSocket"); @@ -201,7 +190,7 @@ describe("Google Live Video Talk", () => { }, }); expect(getUserMedia).toHaveBeenNthCalledWith(2, { video: true }); - expect(onVideoStream).toHaveBeenCalledWith(expect.any(TestMediaStream)); + expect(onVideoStream).toHaveBeenCalledWith(camera); expect(onStatus).toHaveBeenCalledWith("listening"); const countVideoMessages = () => @@ -212,9 +201,9 @@ describe("Google Live Video Talk", () => { await vi.advanceTimersByTimeAsync(1); expect(countVideoMessages()).toBe(2); - (videoTrack as { readyState: MediaStreamTrackState }).readyState = "ended"; - await vi.advanceTimersByTimeAsync(1_000); - expect(countVideoMessages()).toBe(2); + await transport.setVideoEnabled(false); + expect(videoStop).toHaveBeenCalledOnce(); + expect(audioStop).not.toHaveBeenCalled(); ws.emitMessage({ toolCall: { functionCalls: [ @@ -231,8 +220,7 @@ describe("Google Live Video Talk", () => { name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, response: { ok: false, - cameraStreamActive: false, - error: "Camera stream is unavailable", + error: "camera is off", }, }, ], @@ -248,6 +236,54 @@ describe("Google Live Video Talk", () => { expect(videoStop).toHaveBeenCalledOnce(); }); + it("clears ended camera state and reacquires on the next enable", async () => { + const audioTrack = { stop: vi.fn() } as unknown as MediaStreamTrack; + const firstVideoTrack = Object.assign(new EventTarget(), { + stop: vi.fn(), + readyState: "live", + enabled: true, + muted: false, + }) as unknown as MediaStreamTrack; + const secondVideoTrack = Object.assign(new EventTarget(), { + stop: vi.fn(), + readyState: "live", + enabled: true, + muted: false, + }) as unknown as MediaStreamTrack; + const audio = { + getAudioTracks: () => [audioTrack], + getTracks: () => [audioTrack], + } as unknown as MediaStream; + const firstCamera = { + getVideoTracks: () => [firstVideoTrack], + getTracks: () => [firstVideoTrack], + } as unknown as MediaStream; + const secondCamera = { + getVideoTracks: () => [secondVideoTrack], + getTracks: () => [secondVideoTrack], + } as unknown as MediaStream; + const getUserMedia = vi + .fn() + .mockResolvedValueOnce(audio) + .mockResolvedValueOnce(firstCamera) + .mockResolvedValueOnce(secondCamera); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined); + const onVideoStream = vi.fn(); + const transport = createTransport({ onVideoStream }); + + await transport.start(); + await transport.setVideoEnabled(true); + firstVideoTrack.dispatchEvent(new Event("ended")); + + expect(onVideoStream).toHaveBeenLastCalledWith(null); + await transport.setVideoEnabled(true); + expect(getUserMedia).toHaveBeenNthCalledWith(3, { video: true }); + expect(onVideoStream).toHaveBeenLastCalledWith(secondCamera); + + transport.stop(); + }); + it("releases acquired media when stopped during the camera prompt", async () => { const audioStop = vi.fn(); const videoStop = vi.fn(); @@ -267,14 +303,15 @@ describe("Google Live Video Talk", () => { vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); const transport = createTransport({}); - const start = transport.start(); - await Promise.resolve(); + await transport.start(); + const enabling = transport.setVideoEnabled(true); + await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledTimes(2)); transport.stop(); resolveCamera(camera); - await start; + await enabling; expect(audioStop).toHaveBeenCalledOnce(); expect(videoStop).toHaveBeenCalledOnce(); - expect(FakeGoogleLiveWebSocket.instance).toBeUndefined(); + expect(FakeGoogleLiveWebSocket.instance?.readyState).toBe(3); }); }); diff --git a/ui/src/pages/chat/realtime-talk-google-live.ts b/ui/src/pages/chat/realtime-talk-google-live.ts index 0eff349085e..9e96c1be18e 100644 --- a/ui/src/pages/chat/realtime-talk-google-live.ts +++ b/ui/src/pages/chat/realtime-talk-google-live.ts @@ -8,7 +8,7 @@ import { RealtimeTalkPcmInputPump, RealtimeTalkPcmOutputQueue, } from "./realtime-talk-audio.ts"; -import { openRealtimeTalkInput } from "./realtime-talk-input.ts"; +import { openRealtimeTalkCamera, openRealtimeTalkInput } from "./realtime-talk-input.ts"; import type { RealtimeTalkJsonPcmWebSocketSessionResult } from "./realtime-talk-shared.ts"; import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, @@ -106,6 +106,7 @@ function buildGoogleLiveUrl(session: RealtimeTalkJsonPcmWebSocketSessionResult): export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { private ws: WebSocket | null = null; private media: MediaStream | null = null; + private cameraMedia: MediaStream | null = null; private captureVideo: HTMLVideoElement | null = null; private inputContext: AudioContext | null = null; private outputContext: AudioContext | null = null; @@ -113,6 +114,9 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { private readonly inputPump = new RealtimeTalkPcmInputPump(); private closed = false; private mediaSetupController: AbortController | null = null; + private cameraSetupController: AbortController | null = null; + private readonly handleCameraTrackEnded = () => this.releaseCamera(); + private setupComplete = false; private videoFramesActive = false; private hasSentVideoFrame = false; private videoFrameTimer: ReturnType | null = null; @@ -143,7 +147,6 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { let media: MediaStream; try { media = await openRealtimeTalkInput(this.ctx.inputDeviceId, { - video: this.ctx.videoEnabled, signal: mediaSetupController.signal, }); } catch (error) { @@ -161,16 +164,6 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { return; } this.media = media; - if (this.ctx.videoEnabled) { - const captureVideo = document.createElement("video"); - captureVideo.autoplay = true; - captureVideo.muted = true; - captureVideo.playsInline = true; - captureVideo.srcObject = media; - this.captureVideo = captureVideo; - this.ctx.callbacks.onVideoStream?.(media); - void captureVideo.play().catch(() => undefined); - } this.inputContext = new AudioContext({ sampleRate: this.session.audio.inputSampleRateHz }); this.outputContext = new AudioContext({ sampleRate: this.session.audio.outputSampleRateHz }); if (this.ctx.callbacks.onInputLevel) { @@ -201,6 +194,57 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { }); } + async setVideoEnabled(enabled: boolean): Promise { + if (!enabled) { + this.releaseCamera(); + return; + } + if (this.closed) { + throw new Error("Realtime Talk session is closed"); + } + if (this.cameraMedia?.getVideoTracks().some((track) => track.readyState === "live")) { + return; + } + this.cameraSetupController?.abort(); + const controller = new AbortController(); + this.cameraSetupController = controller; + let camera: MediaStream; + try { + camera = await openRealtimeTalkCamera(controller.signal); + } catch (error) { + if (this.closed || controller.signal.aborted) { + return; + } + throw error; + } finally { + if (this.cameraSetupController === controller) { + this.cameraSetupController = null; + } + } + if (this.closed || controller.signal.aborted) { + camera.getTracks().forEach((track) => track.stop()); + return; + } + this.cameraMedia = camera; + // External track loss clears preview state so the next toggle reacquires the camera. + camera + .getVideoTracks() + .forEach((track) => + track.addEventListener("ended", this.handleCameraTrackEnded, { once: true }), + ); + const captureVideo = document.createElement("video"); + captureVideo.autoplay = true; + captureVideo.muted = true; + captureVideo.playsInline = true; + captureVideo.srcObject = camera; + this.captureVideo = captureVideo; + this.ctx.callbacks.onVideoStream?.(camera); + void captureVideo.play().catch(() => undefined); + if (this.setupComplete) { + this.startVideoFrames(); + } + } + stop(): void { if (!this.closed) { this.emitTalkEvent({ type: "session.closed", final: true }); @@ -208,7 +252,9 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { this.closed = true; this.mediaSetupController?.abort(); this.mediaSetupController = null; - this.stopVideoFrames(); + this.cameraSetupController?.abort(); + this.cameraSetupController = null; + this.setupComplete = false; for (const controller of this.consultAbortControllers) { controller.abort(); } @@ -219,11 +265,7 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { this.inputMeter = null; this.media?.getTracks().forEach((track) => track.stop()); this.media = null; - if (this.captureVideo) { - this.captureVideo.srcObject = null; - this.captureVideo = null; - } - this.ctx.callbacks.onVideoStream?.(null); + this.releaseCamera(); this.stopOutput(); void this.inputContext?.close(); this.inputContext = null; @@ -275,6 +317,7 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { return; } if (message.setupComplete) { + this.setupComplete = true; this.ctx.callbacks.onStatus?.("listening"); this.emitTalkEvent({ type: "session.ready" }); this.startVideoFrames(); @@ -395,11 +438,10 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { } if (name === REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME) { const active = this.videoFramesActive && this.hasSentVideoFrame && this.isCameraTrackUsable(); - this.submitToolResult(callId, { - ok: active, - cameraStreamActive: active, - ...(!active ? { error: "Camera stream is unavailable" } : {}), - }); + this.submitToolResult( + callId, + active ? { ok: true, cameraStreamActive: true } : { ok: false, error: "camera is off" }, + ); this.emitTalkEvent({ type: active ? "tool.result" : "tool.error", callId, @@ -488,7 +530,7 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { } private startVideoFrames(): void { - if (!this.ctx.videoEnabled || !this.captureVideo || this.videoFramesActive || this.closed) { + if (!this.captureVideo || this.videoFramesActive || this.closed) { return; } this.videoFramesActive = true; @@ -547,17 +589,33 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { } private hasLiveCameraTrack(): boolean { - return this.media?.getVideoTracks().some((track) => track.readyState === "live") === true; + return this.cameraMedia?.getVideoTracks().some((track) => track.readyState === "live") === true; } private isCameraTrackUsable(): boolean { return ( - this.media + this.cameraMedia ?.getVideoTracks() .some((track) => track.readyState === "live" && track.enabled && !track.muted) === true ); } + private releaseCamera(): void { + this.cameraSetupController?.abort(); + this.cameraSetupController = null; + this.stopVideoFrames(); + this.cameraMedia?.getVideoTracks().forEach((track) => { + track.removeEventListener("ended", this.handleCameraTrackEnded); + track.stop(); + }); + this.cameraMedia = null; + if (this.captureVideo) { + this.captureVideo.srcObject = null; + this.captureVideo = null; + } + this.ctx.callbacks.onVideoStream?.(null); + } + private sendControlSpeechMessage(message: string): void { this.stopOutput(); if (!isGemini31LiveModel(this.session.model)) { diff --git a/ui/src/pages/chat/realtime-talk-input.test.ts b/ui/src/pages/chat/realtime-talk-input.test.ts index 935dfe44220..2de499bed59 100644 --- a/ui/src/pages/chat/realtime-talk-input.test.ts +++ b/ui/src/pages/chat/realtime-talk-input.test.ts @@ -1,6 +1,10 @@ // @vitest-environment node import { afterEach, describe, expect, it, vi } from "vitest"; -import { discoverRealtimeTalkInputs, openRealtimeTalkInput } from "./realtime-talk-input.ts"; +import { + discoverRealtimeTalkInputs, + openRealtimeTalkCamera, + openRealtimeTalkInput, +} from "./realtime-talk-input.ts"; function mediaDevice(kind: MediaDeviceKind, deviceId: string, label: string): MediaDeviceInfo { return { kind, deviceId, label, groupId: "", toJSON: () => ({}) } as MediaDeviceInfo; @@ -115,23 +119,14 @@ describe("realtime Talk microphone inputs", () => { }); }); - it("acquires camera separately so video errors do not mask microphone errors", async () => { - const audioTrack = {} as MediaStreamTrack; - const videoTrack = {} as MediaStreamTrack; - const audio = { - getAudioTracks: () => [audioTrack], - getTracks: () => [audioTrack], - } as unknown as MediaStream; - const camera = { getVideoTracks: () => [videoTrack] } as unknown as MediaStream; - class TestMediaStream { - constructor(readonly tracks: MediaStreamTrack[]) {} - } + it("acquires camera separately so camera errors cannot stop microphone input", async () => { + const audio = { getTracks: () => [] } as unknown as MediaStream; + const camera = { getTracks: () => [] } as unknown as MediaStream; const getUserMedia = vi.fn().mockResolvedValueOnce(audio).mockResolvedValueOnce(camera); vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); - vi.stubGlobal("MediaStream", TestMediaStream); - const combined = await openRealtimeTalkInput("usb-mic", { video: true }); - expect(combined).toBeInstanceOf(TestMediaStream); + await expect(openRealtimeTalkInput("usb-mic")).resolves.toBe(audio); + await expect(openRealtimeTalkCamera()).resolves.toBe(camera); expect(getUserMedia).toHaveBeenNthCalledWith(1, { audio: { autoGainControl: true, @@ -141,65 +136,38 @@ describe("realtime Talk microphone inputs", () => { }, }); expect(getUserMedia).toHaveBeenNthCalledWith(2, { video: true }); - expect((combined as unknown as TestMediaStream).tracks).toEqual([audioTrack, videoTrack]); }); it("reports camera permission denial with actionable guidance", async () => { - const stop = vi.fn(); - const audio = { getTracks: () => [{ stop }] } as unknown as MediaStream; - const getUserMedia = vi - .fn() - .mockResolvedValueOnce(audio) - .mockRejectedValueOnce(new DOMException("denied", "NotAllowedError")); + const getUserMedia = vi.fn().mockRejectedValue(new DOMException("denied", "NotAllowedError")); vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); - await expect(openRealtimeTalkInput(undefined, { video: true })).rejects.toThrow( - "Camera access is blocked", - ); - expect(stop).toHaveBeenCalledOnce(); + await expect(openRealtimeTalkCamera()).rejects.toThrow("Camera access is blocked"); }); - it("reports a missing camera when an exact microphone is selected", async () => { - const stop = vi.fn(); - const audio = { getTracks: () => [{ stop }] } as unknown as MediaStream; - const getUserMedia = vi - .fn() - .mockResolvedValueOnce(audio) - .mockRejectedValueOnce(new DOMException("missing", "NotFoundError")); + it("reports a missing camera", async () => { + const getUserMedia = vi.fn().mockRejectedValue(new DOMException("missing", "NotFoundError")); vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); - await expect(openRealtimeTalkInput("usb-mic", { video: true })).rejects.toThrow( - "No camera was found", - ); - expect(stop).toHaveBeenCalledOnce(); + await expect(openRealtimeTalkCamera()).rejects.toThrow("No camera was found"); }); - it("releases partial media when video acquisition is cancelled", async () => { - const audioStop = vi.fn(); + it("releases camera media when acquisition is cancelled", async () => { const videoStop = vi.fn(); - const audio = { - getAudioTracks: () => [{} as MediaStreamTrack], - getTracks: () => [{ stop: audioStop }], - } as unknown as MediaStream; const camera = { - getVideoTracks: () => [{} as MediaStreamTrack], getTracks: () => [{ stop: videoStop }], } as unknown as MediaStream; let resolveCamera: (stream: MediaStream) => void = () => undefined; const cameraPending = new Promise((resolve) => { resolveCamera = resolve; }); - const getUserMedia = vi.fn().mockResolvedValueOnce(audio).mockReturnValueOnce(cameraPending); + const getUserMedia = vi.fn().mockReturnValue(cameraPending); vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); const controller = new AbortController(); - const opening = openRealtimeTalkInput(undefined, { - video: true, - signal: controller.signal, - }); - await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledTimes(2)); + const opening = openRealtimeTalkCamera(controller.signal); + await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledOnce()); controller.abort(); - expect(audioStop).toHaveBeenCalledOnce(); resolveCamera(camera); await expect(opening).rejects.toMatchObject({ name: "AbortError" }); diff --git a/ui/src/pages/chat/realtime-talk-input.ts b/ui/src/pages/chat/realtime-talk-input.ts index 8e95e7e4fe4..3186ef9733d 100644 --- a/ui/src/pages/chat/realtime-talk-input.ts +++ b/ui/src/pages/chat/realtime-talk-input.ts @@ -105,7 +105,7 @@ function realtimeTalkAbortReason(signal: AbortSignal): Error { export async function openRealtimeTalkInput( inputDeviceId: string | undefined, - options: { video?: boolean; signal?: AbortSignal } = {}, + options: { signal?: AbortSignal } = {}, ): Promise { const devices = globalThis.navigator?.mediaDevices; if (!devices?.getUserMedia) { @@ -130,31 +130,25 @@ export async function openRealtimeTalkInput( audio.getTracks().forEach((track) => track.stop()); throw realtimeTalkAbortReason(options.signal); } - if (!options.video) { - return audio; - } + return audio; +} - let audioStopped = false; - const stopAudio = () => { - if (audioStopped) { - return; - } - audioStopped = true; - audio.getTracks().forEach((track) => track.stop()); - }; - options.signal?.addEventListener("abort", stopAudio, { once: true }); - let camera: MediaStream | undefined; +export async function openRealtimeTalkCamera(signal?: AbortSignal): Promise { + const devices = globalThis.navigator?.mediaDevices; + if (!devices?.getUserMedia) { + throw new Error(t("chat.composer.cameraAccessFailed")); + } + let camera: MediaStream; try { camera = await devices.getUserMedia({ video: true }); - if (options.signal?.aborted) { - throw realtimeTalkAbortReason(options.signal); + if (signal?.aborted) { + camera.getTracks().forEach((track) => track.stop()); + throw realtimeTalkAbortReason(signal); } - return new MediaStream([...audio.getAudioTracks(), ...camera.getVideoTracks()]); + return camera; } catch (error) { - camera?.getTracks().forEach((track) => track.stop()); - stopAudio(); - if (options.signal?.aborted) { - throw realtimeTalkAbortReason(options.signal); + if (signal?.aborted) { + throw realtimeTalkAbortReason(signal); } if (error instanceof DOMException && error.name === "NotAllowedError") { throw new Error(t("chat.composer.cameraPermissionBlocked"), { cause: error }); @@ -165,8 +159,6 @@ export async function openRealtimeTalkInput( if (error instanceof DOMException && error.name === "NotReadableError") { throw new Error(t("chat.composer.cameraBusy"), { cause: error }); } - throw error; - } finally { - options.signal?.removeEventListener("abort", stopAudio); + throw new Error(t("chat.composer.cameraAccessFailed"), { cause: error }); } } diff --git a/ui/src/pages/chat/realtime-talk-shared.ts b/ui/src/pages/chat/realtime-talk-shared.ts index f202f3b8d97..41cb390827c 100644 --- a/ui/src/pages/chat/realtime-talk-shared.ts +++ b/ui/src/pages/chat/realtime-talk-shared.ts @@ -16,6 +16,7 @@ export type RealtimeTalkEvent = TalkEvent; export type RealtimeTalkCallbacks = { onStatus?: (status: RealtimeTalkStatus, detail?: string) => void; + onVideoCapability?: (capable: boolean) => void; onInputLevel?: (level: number) => void; onTranscript?: (entry: { role: "user" | "assistant"; text: string; final: boolean }) => void; onTalkEvent?: (event: RealtimeTalkEvent) => void; @@ -101,6 +102,7 @@ export type RealtimeTalkSessionResult = export type RealtimeTalkTransport = { start(): Promise; stop(): void; + setVideoEnabled?: (enabled: boolean) => Promise; }; export type RealtimeTalkTransportContext = { @@ -108,7 +110,6 @@ export type RealtimeTalkTransportContext = { sessionKey: string; callbacks: RealtimeTalkCallbacks; inputDeviceId?: string; - videoEnabled?: boolean; consultThinkingLevel?: string; consultFastMode?: boolean; }; diff --git a/ui/src/pages/chat/realtime-talk-webrtc-video.test.ts b/ui/src/pages/chat/realtime-talk-webrtc-video.test.ts index ea2aafdc163..aee6e0fd60c 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc-video.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc-video.test.ts @@ -67,33 +67,24 @@ describe("OpenAI Realtime Video Talk", () => { vi.restoreAllMocks(); }); - it("keeps camera local, bounds the frame event, and releases both tracks", async () => { + it("starts audio-only, toggles local camera, and reports camera-off tool calls", async () => { const audioStop = vi.fn(); const videoStop = vi.fn(); const audioTrack = { stop: audioStop } as unknown as MediaStreamTrack; - const videoTrack = { stop: videoStop } as unknown as MediaStreamTrack; + const videoTrack = Object.assign(new EventTarget(), { + stop: videoStop, + readyState: "live", + }) as unknown as MediaStreamTrack; const audio = { getAudioTracks: () => [audioTrack], getTracks: () => [audioTrack], } as unknown as MediaStream; const camera = { getVideoTracks: () => [videoTrack], + getTracks: () => [videoTrack], } as unknown as MediaStream; - class TestMediaStream { - constructor(readonly tracks: MediaStreamTrack[]) {} - getAudioTracks() { - return [audioTrack]; - } - getVideoTracks() { - return [videoTrack]; - } - getTracks() { - return this.tracks; - } - } const getUserMedia = vi.fn().mockResolvedValueOnce(audio).mockResolvedValueOnce(camera); vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); - vi.stubGlobal("MediaStream", TestMediaStream); const originalCreateElement = document.createElement.bind(document); let videoReadyState: number = HTMLMediaElement.HAVE_METADATA; @@ -129,13 +120,17 @@ describe("OpenAI Realtime Video Talk", () => { client: {} as never, sessionKey: "main", callbacks: { onStatus, onTalkEvent, onVideoStream }, - videoEnabled: true, }, ); await transport.start(); const peer = FakePeerConnection.instance; - const combined = onVideoStream.mock.calls[0]?.[0] as MediaStream; + expect(getUserMedia).toHaveBeenCalledOnce(); + expect(peer?.addTrack).toHaveBeenCalledWith(audioTrack, audio); + expect(onVideoStream).not.toHaveBeenCalled(); + + await transport.setVideoEnabled(true); + expect(onVideoStream).toHaveBeenCalledWith(camera); peer?.channel.dispatchEvent( new MessageEvent("message", { data: JSON.stringify({ @@ -184,24 +179,129 @@ describe("OpenAI Realtime Video Talk", () => { }); expect(getUserMedia).toHaveBeenNthCalledWith(2, { video: true }); expect(peer?.addTrack).toHaveBeenCalledOnce(); - expect(peer?.addTrack).toHaveBeenCalledWith(audioTrack, combined); - expect(combined).toBeInstanceOf(TestMediaStream); - expect(onVideoStream).toHaveBeenCalledWith(combined); expect(drawImage).toHaveBeenCalledTimes(2); expect(onTalkEvent.mock.calls.map(([event]) => event.type)).toContain("tool.result"); for (const [payload] of peer?.channel.send.mock.calls ?? []) { expect(new TextEncoder().encode(String(payload)).length).toBeLessThanOrEqual(512); } - peer!.connectionState = "failed"; - peer!.dispatchEvent(new Event("connectionstatechange")); - expect(onStatus).toHaveBeenLastCalledWith("error", "Realtime connection closed"); + await transport.setVideoEnabled(false); expect(onVideoStream).toHaveBeenLastCalledWith(null); - expect(audioStop).toHaveBeenCalledOnce(); expect(videoStop).toHaveBeenCalledOnce(); + expect(audioStop).not.toHaveBeenCalled(); + + peer?.channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ + type: "response.function_call_arguments.done", + item_id: "item-camera-off", + call_id: "call-camera-off", + name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, + arguments: "{}", + }), + }), + ); + await vi.waitFor(() => + expect(sentRealtimeEvents()).toContainEqual({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-camera-off", + output: JSON.stringify({ ok: false, error: "camera is off" }), + }, + }), + ); + expect(onStatus).not.toHaveBeenCalledWith("error", expect.anything()); + + transport.stop(); + expect(audioStop).toHaveBeenCalledOnce(); }); - it("releases the microphone when stopped during the camera prompt", async () => { + it("clears ended camera state and reacquires on the next enable", async () => { + const audioTrack = { stop: vi.fn() } as unknown as MediaStreamTrack; + const firstVideoTrack = Object.assign(new EventTarget(), { + stop: vi.fn(), + readyState: "live", + }) as unknown as MediaStreamTrack; + const secondVideoTrack = Object.assign(new EventTarget(), { + stop: vi.fn(), + readyState: "live", + }) as unknown as MediaStreamTrack; + const audio = { + getAudioTracks: () => [audioTrack], + getTracks: () => [audioTrack], + } as unknown as MediaStream; + const firstCamera = { + getVideoTracks: () => [firstVideoTrack], + getTracks: () => [firstVideoTrack], + } as unknown as MediaStream; + const secondCamera = { + getVideoTracks: () => [secondVideoTrack], + getTracks: () => [secondVideoTrack], + } as unknown as MediaStream; + const getUserMedia = vi + .fn() + .mockResolvedValueOnce(audio) + .mockResolvedValueOnce(firstCamera) + .mockResolvedValueOnce(secondCamera); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined); + const onVideoStream = vi.fn(); + const transport = new WebRtcSdpRealtimeTalkTransport( + { provider: "openai", transport: "webrtc", clientSecret: "test-client-secret" }, + { + client: {} as never, + sessionKey: "main", + callbacks: { onVideoStream }, + }, + ); + + await transport.start(); + await transport.setVideoEnabled(true); + firstVideoTrack.dispatchEvent(new Event("ended")); + + expect(onVideoStream).toHaveBeenLastCalledWith(null); + await transport.setVideoEnabled(true); + expect(getUserMedia).toHaveBeenNthCalledWith(3, { video: true }); + expect(onVideoStream).toHaveBeenLastCalledWith(secondCamera); + + transport.stop(); + }); + + it("keeps voice alive when lazy camera acquisition fails", async () => { + const audioStop = vi.fn(); + const audioTrack = { stop: audioStop } as unknown as MediaStreamTrack; + const audio = { + getAudioTracks: () => [audioTrack], + getTracks: () => [audioTrack], + } as unknown as MediaStream; + const getUserMedia = vi + .fn() + .mockResolvedValueOnce(audio) + .mockRejectedValueOnce(new DOMException("denied", "NotAllowedError")); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + const onStatus = vi.fn(); + const onVideoStream = vi.fn(); + const transport = new WebRtcSdpRealtimeTalkTransport( + { provider: "openai", transport: "webrtc", clientSecret: "test-client-secret" }, + { + client: {} as never, + sessionKey: "main", + callbacks: { onStatus, onVideoStream }, + }, + ); + + await transport.start(); + await expect(transport.setVideoEnabled(true)).rejects.toThrow("Camera access is blocked"); + + expect(audioStop).not.toHaveBeenCalled(); + expect(onVideoStream).not.toHaveBeenCalled(); + expect(onStatus).not.toHaveBeenCalledWith("error", expect.anything()); + transport.stop(); + expect(audioStop).toHaveBeenCalledOnce(); + }); + + it("releases acquired media when stopped during the camera prompt", async () => { const audioStop = vi.fn(); const videoStop = vi.fn(); const audio = { @@ -228,17 +328,17 @@ describe("OpenAI Realtime Video Talk", () => { client: {} as never, sessionKey: "main", callbacks: {}, - videoEnabled: true, }, ); - const starting = transport.start(); + await transport.start(); + const enabling = transport.setVideoEnabled(true); await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledTimes(2)); transport.stop(); expect(audioStop).toHaveBeenCalledOnce(); resolveCamera(camera); - await expect(starting).resolves.toBeUndefined(); + await expect(enabling).resolves.toBeUndefined(); expect(videoStop).toHaveBeenCalledOnce(); }); }); diff --git a/ui/src/pages/chat/realtime-talk-webrtc.test.ts b/ui/src/pages/chat/realtime-talk-webrtc.test.ts index 1cb4f67d40a..2d8eb51532b 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.test.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.test.ts @@ -84,7 +84,6 @@ function createOpenAiTransport( client: Record = {}, callbacks: Record = {}, inputDeviceId?: string, - videoEnabled?: boolean, ): WebRtcSdpRealtimeTalkTransport { return new WebRtcSdpRealtimeTalkTransport( { @@ -97,7 +96,6 @@ function createOpenAiTransport( sessionKey: "main", callbacks: callbacks as never, inputDeviceId, - videoEnabled, }, ); } diff --git a/ui/src/pages/chat/realtime-talk-webrtc.ts b/ui/src/pages/chat/realtime-talk-webrtc.ts index a79fd949db3..40c4c2ff355 100644 --- a/ui/src/pages/chat/realtime-talk-webrtc.ts +++ b/ui/src/pages/chat/realtime-talk-webrtc.ts @@ -1,7 +1,7 @@ // Control UI chat module implements realtime talk webrtc behavior. import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../../../src/talk/describe-view-tool.js"; import { RealtimeTalkMediaStreamMeter } from "./realtime-talk-audio.ts"; -import { openRealtimeTalkInput } from "./realtime-talk-input.ts"; +import { openRealtimeTalkCamera, openRealtimeTalkInput } from "./realtime-talk-input.ts"; import type { RealtimeTalkWebRtcSdpSessionResult } from "./realtime-talk-shared.ts"; import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, @@ -53,6 +53,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { private peer: RTCPeerConnection | null = null; private channel: RTCDataChannel | null = null; private media: MediaStream | null = null; + private cameraMedia: MediaStream | null = null; private audio: HTMLAudioElement | null = null; private captureVideo: HTMLVideoElement | null = null; private inputMeter: RealtimeTalkMediaStreamMeter | null = null; @@ -63,6 +64,8 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { private toolBuffers = new Map(); private pendingOfferRequest: PendingOfferRequest | null = null; private mediaSetupController: AbortController | null = null; + private cameraSetupController: AbortController | null = null; + private readonly handleCameraTrackEnded = () => this.releaseCamera(); private readonly consultAbortControllers = new Set(); private readonly emitTalkEvent: ReturnType; @@ -98,7 +101,6 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { media = await this.awaitSetupStep( peer, openRealtimeTalkInput(this.ctx.inputDeviceId, { - video: this.ctx.videoEnabled, signal: mediaSetupController.signal, }), ); @@ -115,16 +117,6 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { return; } this.media = media; - if (this.ctx.videoEnabled) { - const captureVideo = document.createElement("video"); - captureVideo.autoplay = true; - captureVideo.muted = true; - captureVideo.playsInline = true; - captureVideo.srcObject = media; - this.captureVideo = captureVideo; - this.ctx.callbacks.onVideoStream?.(media); - void captureVideo.play().catch(() => undefined); - } if (this.ctx.callbacks.onInputLevel) { this.inputMeter = new RealtimeTalkMediaStreamMeter(this.ctx.callbacks.onInputLevel); this.inputMeter.start(media); @@ -184,6 +176,54 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { ); } + async setVideoEnabled(enabled: boolean): Promise { + if (!enabled) { + this.releaseCamera(); + return; + } + if (this.closed) { + throw new Error("Realtime Talk session is closed"); + } + if (this.cameraMedia?.getVideoTracks().some((track) => track.readyState === "live")) { + return; + } + this.cameraSetupController?.abort(); + const controller = new AbortController(); + this.cameraSetupController = controller; + let camera: MediaStream; + try { + camera = await openRealtimeTalkCamera(controller.signal); + } catch (error) { + if (this.closed || controller.signal.aborted) { + return; + } + throw error; + } finally { + if (this.cameraSetupController === controller) { + this.cameraSetupController = null; + } + } + if (this.closed || controller.signal.aborted) { + camera.getTracks().forEach((track) => track.stop()); + return; + } + this.cameraMedia = camera; + // External track loss clears preview state so the next toggle reacquires the camera. + camera + .getVideoTracks() + .forEach((track) => + track.addEventListener("ended", this.handleCameraTrackEnded, { once: true }), + ); + const captureVideo = document.createElement("video"); + captureVideo.autoplay = true; + captureVideo.muted = true; + captureVideo.playsInline = true; + captureVideo.srcObject = camera; + this.captureVideo = captureVideo; + this.ctx.callbacks.onVideoStream?.(camera); + void captureVideo.play().catch(() => undefined); + } + private async readOfferAnswer( peer: RTCPeerConnection, offer: RTCSessionDescriptionInit, @@ -286,6 +326,8 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { this.closed = true; this.mediaSetupController?.abort(); this.mediaSetupController = null; + this.cameraSetupController?.abort(); + this.cameraSetupController = null; this.abortOfferRequest(); this.channel?.close(); this.channel = null; @@ -293,11 +335,7 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { this.peer = null; this.media?.getTracks().forEach((track) => track.stop()); this.media = null; - if (this.captureVideo) { - this.captureVideo.srcObject = null; - this.captureVideo = null; - } - this.ctx.callbacks.onVideoStream?.(null); + this.releaseCamera(); this.inputMeter?.stop(); this.inputMeter = null; this.audio?.remove(); @@ -526,6 +564,17 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { itemId, payload: { name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME }, }); + if (!this.cameraMedia?.getVideoTracks().some((track) => track.readyState === "live")) { + this.submitToolResult(callId, { ok: false, error: "camera is off" }); + this.emitTalkEvent({ + type: "tool.error", + callId, + itemId, + final: true, + payload: { name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, message: "camera is off" }, + }); + return; + } try { const frame = await captureRealtimeTalkVideoFrame( this.captureVideo, @@ -566,6 +615,21 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport { this.requestResponseCreate(); } + private releaseCamera(): void { + this.cameraSetupController?.abort(); + this.cameraSetupController = null; + this.cameraMedia?.getVideoTracks().forEach((track) => { + track.removeEventListener("ended", this.handleCameraTrackEnded); + track.stop(); + }); + this.cameraMedia = null; + if (this.captureVideo) { + this.captureVideo.srcObject = null; + this.captureVideo = null; + } + this.ctx.callbacks.onVideoStream?.(null); + } + private reportToolResultSubmissionError(error: unknown): void { if (this.closed) { return; diff --git a/ui/src/pages/chat/realtime-talk.test.ts b/ui/src/pages/chat/realtime-talk.test.ts index c936cef3bc4..108aaf6819b 100644 --- a/ui/src/pages/chat/realtime-talk.test.ts +++ b/ui/src/pages/chat/realtime-talk.test.ts @@ -8,6 +8,8 @@ const { relayStop, webRtcStart, webRtcStop, + googleSetVideoEnabled, + webRtcSetVideoEnabled, googleCtor, relayCtor, webRtcCtor, @@ -18,14 +20,16 @@ const { relayStop: vi.fn(), webRtcStart: vi.fn(async () => undefined), webRtcStop: vi.fn(), + googleSetVideoEnabled: vi.fn(async () => undefined), + webRtcSetVideoEnabled: vi.fn(async () => undefined), googleCtor: vi.fn(function () { - return { start: googleStart, stop: googleStop }; + return { start: googleStart, stop: googleStop, setVideoEnabled: googleSetVideoEnabled }; }), relayCtor: vi.fn(function () { return { start: relayStart, stop: relayStop }; }), webRtcCtor: vi.fn(function () { - return { start: webRtcStart, stop: webRtcStop }; + return { start: webRtcStart, stop: webRtcStop, setVideoEnabled: webRtcSetVideoEnabled }; }), })); @@ -51,6 +55,8 @@ describe("RealtimeTalkSession", () => { relayStop.mockClear(); webRtcStart.mockClear(); webRtcStop.mockClear(); + googleSetVideoEnabled.mockClear(); + webRtcSetVideoEnabled.mockClear(); googleCtor.mockClear(); relayCtor.mockClear(); webRtcCtor.mockClear(); @@ -199,6 +205,62 @@ describe("RealtimeTalkSession", () => { expect(relayStart).toHaveBeenCalledTimes(1); }); + it("strips browser capabilities and hides camera when falling back to Gateway relay", async () => { + const request = vi.fn(async (method: string) => { + if (method === "talk.catalog") { + return { + realtime: { + activeProvider: "openai", + providers: [{ id: "openai", label: "OpenAI", supportsVideoFrames: true }], + }, + }; + } + if (method === "talk.client.create") { + throw new Error("browser session unavailable"); + } + if (method === "talk.session.create") { + return { + provider: "openai", + transport: "gateway-relay", + relaySessionId: "relay-1", + audio: { + inputEncoding: "pcm16", + inputSampleRateHz: 24000, + outputEncoding: "pcm16", + outputSampleRateHz: 24000, + }, + }; + } + throw new Error(`Unexpected request: ${method}`); + }); + const onVideoCapability = vi.fn(); + const session = new RealtimeTalkSession( + { request } as never, + "main", + { onVideoCapability }, + { provider: "openai", transport: "gateway-relay" }, + ); + + await session.start(); + + expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", { + sessionKey: "main", + provider: "openai", + transport: "gateway-relay", + capabilities: ["camera-frame"], + }); + expect(request).toHaveBeenNthCalledWith(3, "talk.session.create", { + sessionKey: "main", + provider: "openai", + transport: "gateway-relay", + mode: "realtime", + brain: "agent-consult", + }); + expect(onVideoCapability).toHaveBeenCalledOnce(); + expect(onVideoCapability).toHaveBeenCalledWith(false); + expect(relayCtor).toHaveBeenCalledTimes(1); + }); + it("starts the WebRTC transport for canonical WebRTC sessions", async () => { const request = vi.fn(async () => ({ provider: "openai", @@ -259,6 +321,71 @@ describe("RealtimeTalkSession", () => { ); }); + it("requests camera-frame for the active video-capable provider without enabling camera", async () => { + const request = vi.fn(async (method: string) => { + if (method === "talk.catalog") { + return { + realtime: { + activeProvider: "openai", + providers: [{ id: "openai", label: "OpenAI", supportsVideoFrames: true }], + }, + }; + } + return { + provider: "openai", + transport: "webrtc", + clientSecret: "secret", + }; + }); + const onVideoCapability = vi.fn(); + const session = new RealtimeTalkSession({ request } as never, "main", { + onVideoCapability, + }); + + await session.start(); + + expect(request).toHaveBeenNthCalledWith(1, "talk.catalog", {}); + expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", { + sessionKey: "main", + capabilities: ["camera-frame"], + }); + expect(onVideoCapability).toHaveBeenCalledWith(true); + expect(webRtcCtor).toHaveBeenCalledWith( + expect.any(Object), + expect.not.objectContaining({ videoEnabled: expect.anything() }), + ); + + await session.setVideoEnabled(true); + expect(webRtcSetVideoEnabled).toHaveBeenCalledWith(true); + }); + + it("does not request camera-frame for a provider without video-frame support", async () => { + const request = vi.fn(async (method: string) => { + if (method === "talk.catalog") { + return { + realtime: { + activeProvider: "openai", + providers: [{ id: "openai", label: "OpenAI", supportsVideoFrames: false }], + }, + }; + } + return { + provider: "openai", + transport: "webrtc", + clientSecret: "secret", + }; + }); + const onVideoCapability = vi.fn(); + const session = new RealtimeTalkSession({ request } as never, "main", { + onVideoCapability, + }); + + await session.start(); + + expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", { sessionKey: "main" }); + expect(onVideoCapability).toHaveBeenCalledWith(false); + }); + it("does not fall back to Gateway relay when config selects a client transport", async () => { const clientError = new Error("browser session unavailable"); const request = vi.fn(async (method: string) => { diff --git a/ui/src/pages/chat/realtime-talk.ts b/ui/src/pages/chat/realtime-talk.ts index 1cde137f07d..d897071950a 100644 --- a/ui/src/pages/chat/realtime-talk.ts +++ b/ui/src/pages/chat/realtime-talk.ts @@ -1,4 +1,5 @@ // Control UI chat module implements realtime talk behavior. +import type { TalkCatalogResult } from "@openclaw/gateway-protocol"; import { normalizeTalkTransport } from "../../../../src/talk/talk-session-controller.js"; import type { GatewayBrowserClient } from "../../api/gateway.ts"; import { GatewayRelayRealtimeTalkTransport } from "./realtime-talk-gateway-relay.ts"; @@ -26,12 +27,10 @@ type RealtimeTalkLaunchOptions = { silenceDurationMs?: number; prefixPaddingMs?: number; reasoningEffort?: string; - capabilities?: Array<"camera-frame">; }; type RealtimeTalkLocalOptions = { inputDeviceId?: string; - videoEnabled?: boolean; }; type RealtimeTalkLaunchTransport = NonNullable; @@ -114,7 +113,13 @@ export class RealtimeTalkSession { async start(): Promise { this.closed = false; this.callbacks.onStatus?.("connecting"); - const session = await this.createSession(); + const providerVideoCapable = await this.resolveVideoCapability(); + if (this.closed) { + return; + } + const session = await this.createSession( + providerVideoCapable ? { ...this.options, capabilities: ["camera-frame"] } : this.options, + ); if (this.closed) { return; } @@ -123,24 +128,49 @@ export class RealtimeTalkSession { sessionKey: this.sessionKey, callbacks: this.callbacks, inputDeviceId: this.localOptions.inputDeviceId, - videoEnabled: this.localOptions.videoEnabled, consultThinkingLevel: session.consultThinkingLevel, consultFastMode: session.consultFastMode, }); + this.callbacks.onVideoCapability?.( + providerVideoCapable && typeof this.transport.setVideoEnabled === "function", + ); await this.transport.start(); } - private async createSession(): Promise { + private async resolveVideoCapability(): Promise { + if (!this.callbacks.onVideoCapability) { + return false; + } + try { + const catalog = await this.client.request("talk.catalog", {}); + const selectedProvider = this.options.provider ?? catalog.realtime.activeProvider; + if (!selectedProvider) { + return false; + } + return ( + catalog.realtime.providers.find( + (provider) => + provider.id === selectedProvider || provider.aliases?.includes(selectedProvider), + )?.supportsVideoFrames === true + ); + } catch { + return false; + } + } + + private async createSession( + options: RealtimeTalkLaunchOptions & { capabilities?: Array<"camera-frame"> }, + ): Promise { try { return await this.client.request( "talk.client.create", compactLaunchParams({ sessionKey: this.sessionKey, - ...this.options, + ...options, }), ); } catch (error) { - let transport = this.options.transport; + let transport = options.transport; if (!transport) { let result: RealtimeTalkConfigResult; try { @@ -162,12 +192,14 @@ export class RealtimeTalkSession { if (transport && transport !== "gateway-relay") { throw error; } + const gatewayOptions = { ...options }; + delete gatewayOptions.capabilities; try { return await this.client.request( "talk.session.create", compactLaunchParams({ sessionKey: this.sessionKey, - ...this.options, + ...gatewayOptions, mode: "realtime", transport: transport ?? "gateway-relay", brain: "agent-consult", @@ -185,4 +217,11 @@ export class RealtimeTalkSession { this.transport?.stop(); this.transport = null; } + + async setVideoEnabled(enabled: boolean): Promise { + if (this.closed || !this.transport?.setVideoEnabled) { + throw new Error("Camera is unavailable for this realtime session"); + } + await this.transport.setVideoEnabled(enabled); + } } From 2e21d53c84ec88433dac587bb97f2796e63b20ca Mon Sep 17 00:00:00 2001 From: cxbAsDev Date: Sat, 18 Jul 2026 18:22:30 +0800 Subject: [PATCH 28/76] fix(plugins): bound plugin manifest metadata file reads (#110036) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(plugins): bound plugin manifest metadata file reads * fix(plugins): use correct Error type check in oversized manifest catch readRegularFileSync throws a plain Error (not RangeError) when the file exceeds maxBytes. Change the catch-block type check from RangeError to Error to properly detect oversized plugin manifests and emit the subsystem warning. * test(plugins): exercise valid oversized manifests Co-authored-by: 陈宪彪0668000387 --------- Co-authored-by: Peter Steinberger --- src/plugins/manifest-metadata-scan.test.ts | 51 ++++++++++++++++++++++ src/plugins/manifest-metadata-scan.ts | 21 ++++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/plugins/manifest-metadata-scan.test.ts b/src/plugins/manifest-metadata-scan.test.ts index 8d2de7b8d39..fe5f883e1f2 100644 --- a/src/plugins/manifest-metadata-scan.test.ts +++ b/src/plugins/manifest-metadata-scan.test.ts @@ -126,4 +126,55 @@ describe("listOpenClawPluginManifestMetadata", () => { origin: "global", }); }); + + it("skips oversized plugin manifests to prevent OOM during metadata scan", () => { + const root = createTempRoot(); + const home = path.join(root, "home"); + + const goodPluginDir = path.join(home, ".openclaw", "extensions", "good-plugin"); + writeJson(path.join(goodPluginDir, "openclaw.plugin.json"), { id: "good-plugin" }); + + const oversizedDir = path.join(home, ".openclaw", "extensions", "big-plugin"); + const oversizedPath = path.join(oversizedDir, "openclaw.plugin.json"); + fs.mkdirSync(oversizedDir, { recursive: true }); + fs.writeFileSync( + oversizedPath, + JSON.stringify({ id: "big-plugin", pad: "x".repeat(256 * 1024) }), + "utf8", + ); + expect(fs.statSync(oversizedPath).size).toBeGreaterThan(256 * 1024); + + const records = listOpenClawPluginManifestMetadata({ + OPENCLAW_HOME: home, + OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(root, "empty-bundled"), + }); + + // "good-plugin" is present; "big-plugin" is skipped due to oversized manifest. + expect(records.find((record) => record.manifest.id === "good-plugin")).toBeTruthy(); + expect(records.find((record) => record.manifest.id === "big-plugin")).toBeUndefined(); + }); + + it("accepts plugin manifests at the exact byte limit", () => { + const root = createTempRoot(); + const home = path.join(root, "home"); + + const exactDir = path.join(home, ".openclaw", "extensions", "exact-plugin"); + fs.mkdirSync(exactDir, { recursive: true }); + + // Write a compact JSON manifest padded to exactly the byte limit. + const exactPath = path.join(exactDir, "openclaw.plugin.json"); + const exactManifest = { id: "exact-plugin", pad: "" }; + const compactJson = JSON.stringify(exactManifest); + const requiredPadding = 256 * 1024 - Buffer.byteLength(compactJson, "utf8"); + exactManifest.pad = "x".repeat(requiredPadding); + fs.writeFileSync(exactPath, JSON.stringify(exactManifest), "utf8"); + expect(Buffer.byteLength(fs.readFileSync(exactPath), "utf8")).toBe(256 * 1024); + + const records = listOpenClawPluginManifestMetadata({ + OPENCLAW_HOME: home, + OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(root, "empty-bundled"), + }); + + expect(records.find((record) => record.manifest.id === "exact-plugin")).toBeTruthy(); + }); }); diff --git a/src/plugins/manifest-metadata-scan.ts b/src/plugins/manifest-metadata-scan.ts index 5788e8498d1..d3cf430095c 100644 --- a/src/plugins/manifest-metadata-scan.ts +++ b/src/plugins/manifest-metadata-scan.ts @@ -6,10 +6,18 @@ import { normalizeOptionalString as normalizeTrimmedString } from "@openclaw/nor import { resolveStateDir } from "../config/paths.js"; import { resolveHomeRelativePath } from "../infra/home-dir.js"; import { resolveOpenClawPackageRootSync } from "../infra/openclaw-root.js"; +import { readRegularFileSync } from "../infra/regular-file.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; import { parseJsonWithJson5Fallback } from "../utils/parse-json-compat.js"; import { resolveBundledPluginsDir } from "./bundled-dir.js"; import { readPersistedInstalledPluginIndexSync } from "./installed-plugin-index-store.js"; +// Plugin manifest files are small metadata descriptors. Bound reads to prevent +// a corrupted or hostile manifest from exhausting memory during metadata scan. +const PLUGIN_MANIFEST_METADATA_MAX_BYTES = 256 * 1024; + +const log = createSubsystemLogger("plugins/manifest-metadata-scan"); + type PluginManifestMetadataRecord = { pluginDir: string; manifest: Record; @@ -56,9 +64,18 @@ function listChildPluginDirs( function readJsonObject(filePath: string): Record | undefined { try { - const parsed = parseJsonWithJson5Fallback(fs.readFileSync(filePath, "utf8")); + const { buffer } = readRegularFileSync({ + filePath, + maxBytes: PLUGIN_MANIFEST_METADATA_MAX_BYTES, + }); + const parsed = parseJsonWithJson5Fallback(buffer.toString("utf-8")); return isRecord(parsed) ? parsed : undefined; - } catch { + } catch (err) { + if (err instanceof Error && err.message.includes("exceeds")) { + log.warn( + `Ignoring oversized plugin manifest at ${filePath}: file exceeds the ${PLUGIN_MANIFEST_METADATA_MAX_BYTES}-byte limit`, + ); + } return undefined; } } From 16bd13d2547c434a68e26e1b2de9f876a9af4e95 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 11:25:52 +0100 Subject: [PATCH 29/76] fix(plugin-sdk): refresh channel send API baseline (#110620) --- docs/.generated/plugin-sdk-api-baseline.sha256 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 646b8d4c3c0..001cb093102 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -71,7 +71,7 @@ bba5540be7cf9613a163663decdb2affe2af9bbd3ad7914989ab186f9c2abec1 module/channel 54cdfc60ea96f55453fdccdb1241d8c988aeafe033b9e0cea0f3ecaa13d2e08a module/channel-secret-basic-runtime c78711a2466fca83ae2e68725e61e3a1a4e7dd25abce74d30a943c1be364f704 module/channel-secret-runtime 96e38ca99edb7d39839a7675939d5b8c199f8625502cae15e66b4bc7885922a7 module/channel-secret-tts-runtime -3bb07813120d0d73248991cb4a753434eec0fb8a1ec247ec9f4bb00ef8d1b199 module/channel-send-result +7c90157a95bc0523fc66b1f78ce140f7ec7dbf809dfa3a480244efc01f972754 module/channel-send-result e705b97daac4fbd7f6aeb2e515b1cdaa1db342684644eaef8197767b2595826e module/channel-setup 47719805c7a1d2623dcecafad50abb6cc674262b250e1f0d7020a79ad41cfd42 module/channel-status a4918b82b795f086ae71acb43e2b725fda01f6c8bf2333c155a52581bd4cf8aa module/channel-streaming From e22c2dfaaa890d1d689f894a46c6cc0a4339d301 Mon Sep 17 00:00:00 2001 From: cxbAsDev Date: Sat, 18 Jul 2026 18:27:40 +0800 Subject: [PATCH 30/76] fix(gateway): bound BOOT.md file read size (#101776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(gateway): bound BOOT.md file read size * fix(gateway): treat non-regular BOOT.md as empty/missing and add coverage * fix(gateway): preserve non-size BOOT.md failures while bounding oversized reads * fix(gateway): preserve readable BOOT.md symlink behavior * fix(gateway): treat dangling BOOT.md symlink as missing * fix(gateway): report oversized BOOT files * fix(gateway): preserve missing BOOT race behavior Co-authored-by: 陈宪彪0668000387 --------- Co-authored-by: Peter Steinberger --- src/gateway/boot.test.ts | 75 +++++++++++++++++++++++++++++++++++++++- src/gateway/boot.ts | 26 ++++++++++---- 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/src/gateway/boot.test.ts b/src/gateway/boot.test.ts index bb4a5d2a4d8..6eb9ca43e01 100644 --- a/src/gateway/boot.test.ts +++ b/src/gateway/boot.test.ts @@ -170,7 +170,41 @@ describe("runBootOnce", () => { }); }); - it("returns failed when BOOT.md cannot be read", async () => { + it("skips when BOOT.md disappears after path resolution", async () => { + await withBootWorkspace({ bootContent: "Say hello." }, async (workspaceDir) => { + const bootPath = path.join(workspaceDir, "BOOT.md"); + const realpath = vi.spyOn(fs, "realpath"); + realpath.mockImplementationOnce(async (inputPath) => { + realpath.mockRestore(); + const resolvedPath = await fs.realpath(inputPath); + await fs.rm(resolvedPath); + return resolvedPath; + }); + + await expect(runBootOnce({ cfg: {}, deps: makeDeps(), workspaceDir })).resolves.toEqual({ + status: "skipped", + reason: "missing", + }); + expect(agentCommand).not.toHaveBeenCalled(); + await expect(fs.access(bootPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); + + it("returns failed when BOOT.md exceeds the safe read size limit", async () => { + await withBootWorkspace({ bootContent: "" }, async (workspaceDir) => { + const bootPath = path.join(workspaceDir, "BOOT.md"); + const oversized = Buffer.alloc(16 * 1024 * 1024 + 1, "x"); + await fs.writeFile(bootPath, oversized); + const result = await runBootOnce({ cfg: {}, deps: makeDeps(), workspaceDir }); + expect(result.status).toBe("failed"); + if (result.status === "failed") { + expect(result.reason).toContain("File exceeds 16777216 bytes"); + } + expect(agentCommand).not.toHaveBeenCalled(); + }); + }); + + it("returns failed when BOOT.md is not a regular file", async () => { await withBootWorkspace({ bootAsDirectory: true }, async (workspaceDir) => { const result = await runBootOnce({ cfg: {}, deps: makeDeps(), workspaceDir }); expect(result.status).toBe("failed"); @@ -180,6 +214,45 @@ describe("runBootOnce", () => { expect(agentCommand).not.toHaveBeenCalled(); }); }); + + it("runs agent command when BOOT.md is a symlink to a regular file", async () => { + if (process.platform === "win32") { + // Symlink support in unit tests is not guaranteed on Windows CI runners. + return; + } + await withBootWorkspace({ bootContent: "" }, async (workspaceDir) => { + const bootPath = path.join(workspaceDir, "BOOT.md"); + const targetPath = path.join(workspaceDir, "REAL_BOOT.md"); + await fs.writeFile(targetPath, "Say hello.", "utf-8"); + await fs.rm(bootPath, { force: true }); + await fs.symlink(targetPath, bootPath); + agentCommand.mockResolvedValue(undefined); + await expect(runBootOnce({ cfg: {}, deps: makeDeps(), workspaceDir })).resolves.toEqual({ + status: "ran", + }); + expect(agentCommand).toHaveBeenCalledTimes(1); + const call = requireAgentCall(); + expect(call.message).toContain("Say hello."); + }); + }); + + it("skips when BOOT.md is a dangling symlink", async () => { + if (process.platform === "win32") { + // Symlink support in unit tests is not guaranteed on Windows CI runners. + return; + } + await withBootWorkspace({ bootContent: "" }, async (workspaceDir) => { + const bootPath = path.join(workspaceDir, "BOOT.md"); + const targetPath = path.join(workspaceDir, "MISSING_BOOT.md"); + await fs.rm(bootPath, { force: true }); + await fs.symlink(targetPath, bootPath); + await expect(runBootOnce({ cfg: {}, deps: makeDeps(), workspaceDir })).resolves.toEqual({ + status: "skipped", + reason: "missing", + }); + expect(agentCommand).not.toHaveBeenCalled(); + }); + }); it.each([ { title: "empty", content: " \n", reason: "empty" as const }, { title: "whitespace-only", content: "\n\t ", reason: "empty" as const }, diff --git a/src/gateway/boot.ts b/src/gateway/boot.ts index 9436e63a872..448bfcebe4b 100644 --- a/src/gateway/boot.ts +++ b/src/gateway/boot.ts @@ -21,6 +21,7 @@ import { resolveStorePath } from "../config/sessions/paths.js"; import { preserveTemporarySessionMapping } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatErrorMessage } from "../infra/errors.js"; +import { readRegularFile } from "../infra/regular-file.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { type RuntimeEnv, defaultRuntime } from "../runtime.js"; import { clearBootEchoContextForSession, setBootEchoContextForSession } from "./boot-echo-guard.js"; @@ -72,17 +73,24 @@ function resolveBootSessionKey(sessionKey: string): string { return `agent:${agentId}:boot`; } +const MAX_BOOT_FILE_BYTES = 16 * 1024 * 1024; + async function loadBootFile( workspaceDir: string, ): Promise<{ content?: string; status: "ok" | "missing" | "empty" }> { const bootPath = path.join(workspaceDir, BOOT_FILENAME); + + // Resolve symlinks so BOOT.md can be a readable symlink to a regular file + // while keeping directory/permission/size-limit failures surfaced to the + // operator. ENOENT from either resolution or the bounded open keeps the + // established readFile contract: treat disappearance as missing. + let buffer: Buffer; try { - const content = await fs.readFile(bootPath, "utf-8"); - const trimmed = content.trim(); - if (!trimmed) { - return { status: "empty" }; - } - return { status: "ok", content: trimmed }; + const resolvedPath = await fs.realpath(bootPath); + ({ buffer } = await readRegularFile({ + filePath: resolvedPath, + maxBytes: MAX_BOOT_FILE_BYTES, + })); } catch (err) { const anyErr = err as { code?: string }; if (anyErr.code === "ENOENT") { @@ -90,6 +98,12 @@ async function loadBootFile( } throw err; } + const content = buffer.toString("utf-8"); + const trimmed = content.trim(); + if (!trimmed) { + return { status: "empty" }; + } + return { status: "ok", content: trimmed }; } export async function runBootOnce(params: { From adff355caf2cbd791579f1339c3370adc9d6ef77 Mon Sep 17 00:00:00 2001 From: mushuiyu886 Date: Sat, 18 Jul 2026 18:30:13 +0800 Subject: [PATCH 31/76] fix(plugin-sdk): failed raw channel sends report success (#109906) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(plugin-sdk): reject failed raw channel sends * fix(plugin-sdk): preserve raw result helper compatibility * test(plugin-sdk): cover raw send fallback errors Co-authored-by: 杨浩宇0668001029 * fix(security): bound skill-file audit read via safe reader (restore removed fs dependency) (#110589) --------- Co-authored-by: Peter Steinberger --- src/plugin-sdk/channel-send-result.test.ts | 46 ++++++++++------------ src/plugin-sdk/channel-send-result.ts | 14 ++++++- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/plugin-sdk/channel-send-result.test.ts b/src/plugin-sdk/channel-send-result.test.ts index 19ca5541c85..2cf3ba8770a 100644 --- a/src/plugin-sdk/channel-send-result.test.ts +++ b/src/plugin-sdk/channel-send-result.test.ts @@ -131,38 +131,32 @@ describe("createAttachedChannelResultAdapter", () => { }); describe("createRawChannelSendResultAdapter", () => { - it("normalizes raw send results through adapter methods", async () => { + it("normalizes successes and rejects provider failures", async () => { const adapter = createRawChannelSendResultAdapter({ channel: "zalo", sendText: async () => ({ ok: true, messageId: "m1" }), sendMedia: async () => ({ ok: false, error: "boom" }), }); - const sendCases = [ - { - name: "sendText", - run: () => adapter.sendText!({ cfg: {} as never, to: "x", text: "hi" }), - expected: { - channel: "zalo", - ok: true, - messageId: "m1", - error: undefined, - }, - }, - { - name: "sendMedia", - run: () => adapter.sendMedia!({ cfg: {} as never, to: "x", text: "hi" }), - expected: { - channel: "zalo", - ok: false, - messageId: "", - error: new Error("boom"), - }, - }, - ]; + await expect(adapter.sendText!({ cfg: {} as never, to: "x", text: "hi" })).resolves.toEqual({ + channel: "zalo", + ok: true, + messageId: "m1", + error: undefined, + }); + await expect(adapter.sendMedia!({ cfg: {} as never, to: "x", text: "hi" })).rejects.toThrow( + "boom", + ); + }); - for (const testCase of sendCases) { - await expect(testCase.run()).resolves.toEqual(testCase.expected); - } + it("uses a channel-specific error when a failed result has no message", async () => { + const adapter = createRawChannelSendResultAdapter({ + channel: "legacy-test", + sendText: async () => ({ ok: false }), + }); + + await expect(adapter.sendText!({ cfg: {} as never, to: "x", text: "hi" })).rejects.toThrow( + "Channel send failed for legacy-test", + ); }); }); diff --git a/src/plugin-sdk/channel-send-result.ts b/src/plugin-sdk/channel-send-result.ts index 315aaef3a35..b705c6be870 100644 --- a/src/plugin-sdk/channel-send-result.ts +++ b/src/plugin-sdk/channel-send-result.ts @@ -88,6 +88,16 @@ export function createAttachedChannelResultAdapter(params: { }; } +function buildRawChannelAdapterResult( + channel: string, + result: ChannelSendRawResult, +): OutboundDeliveryResult { + if (!result.ok) { + throw new Error(result.error?.trim() || `Channel send failed for ${channel}`); + } + return buildChannelSendResult(channel, result); +} + /** Wraps legacy raw text/media send methods and normalizes their results. */ export function createRawChannelSendResultAdapter(params: { /** Channel id attached to every normalized legacy send result. */ @@ -99,10 +109,10 @@ export function createRawChannelSendResultAdapter(params: { }): Pick { return { sendText: params.sendText - ? async (ctx) => buildChannelSendResult(params.channel, await params.sendText!(ctx)) + ? async (ctx) => buildRawChannelAdapterResult(params.channel, await params.sendText!(ctx)) : undefined, sendMedia: params.sendMedia - ? async (ctx) => buildChannelSendResult(params.channel, await params.sendMedia!(ctx)) + ? async (ctx) => buildRawChannelAdapterResult(params.channel, await params.sendMedia!(ctx)) : undefined, }; } From cc920838bc3d857cba373acc0deb2d0b9e21e4ba Mon Sep 17 00:00:00 2001 From: "Chris M." Date: Sat, 18 Jul 2026 03:33:57 -0700 Subject: [PATCH 32/76] fix(android): recover Wear requests from stale phone routes (#110423) * fix(android): harden Wear phone routing * fix(android): preserve proven Wear phone routes Co-authored-by: Chris Mish * fix(android): guard Wear discovery invalidation Co-authored-by: Chris Mish --------- Co-authored-by: Chris Mish Co-authored-by: Peter Steinberger --- .../java/ai/openclaw/wear/WearProxyClient.kt | 196 ++++++--- .../openclaw/wear/WearProxyListenerService.kt | 18 +- .../ai/openclaw/wear/WearProxyClientTest.kt | 371 ++++++++++++++++++ 3 files changed, 523 insertions(+), 62 deletions(-) diff --git a/apps/android/wear/src/main/java/ai/openclaw/wear/WearProxyClient.kt b/apps/android/wear/src/main/java/ai/openclaw/wear/WearProxyClient.kt index a9aa798eba5..fd7eea770af 100644 --- a/apps/android/wear/src/main/java/ai/openclaw/wear/WearProxyClient.kt +++ b/apps/android/wear/src/main/java/ai/openclaw/wear/WearProxyClient.kt @@ -12,7 +12,6 @@ import com.google.android.gms.wearable.CapabilityClient import com.google.android.gms.wearable.Wearable import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.ensureActive @@ -21,13 +20,12 @@ import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withTimeoutOrNull import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.buildJsonObject import java.util.UUID import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicReference import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException @@ -77,10 +75,9 @@ internal class WearProxyClient private constructor( private val transport: WearMessageTransport, ) : WearRpcRequester { private val pending = ConcurrentHashMap() - private val selectedPhoneNodeId = AtomicReference() private val preferredPhoneLock = Any() - private var preferredPhoneKnown = false - private var preferredPhoneNodeId: String? = null + private var preferredPhoneGeneration = 0L + private var registeredPhone: PreferredPhoneRegistration? = null private val inboundMutex = Mutex() private val mutableEvents = MutableSharedFlow( @@ -98,31 +95,49 @@ internal class WearProxyClient private constructor( params: JsonObject, expectedNodeId: String?, requirePreferredNode: Boolean, - ): WearRpcResult = - try { - withTimeout(REQUEST_TIMEOUT_MS) { - requestBeforeDeadline(method, params, expectedNodeId, requirePreferredNode) + ): WearRpcResult { + var attemptedPreferredPhone: PreferredPhoneRegistration? = null + val result = + withTimeoutOrNull(REQUEST_TIMEOUT_MS) { + requestBeforeDeadline(method, params, expectedNodeId, requirePreferredNode) { registration -> + attemptedPreferredPhone = registration + } } - } catch (_: TimeoutCancellationException) { - throw WearProxyException("timeout", "Paired phone did not respond") - } + if (result != null) return result + + // MessageClient success only proves the request was queued. A silent phone + // must be rediscovered just like a node that rejected the send outright. + invalidatePreferredPhone(attemptedPreferredPhone) + throw WearProxyException("timeout", "Paired phone did not respond") + } private suspend fun requestBeforeDeadline( method: WearRpcMethod, params: JsonObject, expectedNodeId: String?, requirePreferredNode: Boolean, + recordPreferredPhoneAttempt: (PreferredPhoneRegistration?) -> Unit, ): WearRpcResult { // Stateful RPCs stay on the phone that supplied their session/transcript. // Rediscovery here could route a shared session key to a different phone. - val requiredPreferredNodeId = if (requirePreferredNode) resolvePreferredPhoneNode() else null - val nodeId = expectedNodeId ?: requiredPreferredNodeId ?: resolvePreferredPhoneNode() - if (requirePreferredNode && expectedNodeId != null && requiredPreferredNodeId != expectedNodeId) { + val preferredPhone = + when { + requirePreferredNode || expectedNodeId == null -> resolvePreferredPhone() + else -> preferredPhoneRegistration(expectedNodeId) + } + val nodeId = expectedNodeId ?: checkNotNull(preferredPhone).nodeId + if (requirePreferredNode && expectedNodeId != null && preferredPhone?.nodeId != expectedNodeId) { throw WearProxyException("phone_changed", "Preferred phone changed during request") } + recordPreferredPhoneAttempt(preferredPhone?.takeIf { it.nodeId == nodeId }) val requestId = UUID.randomUUID().toString() val response = CompletableDeferred() - val pendingRequest = PendingWearRequest(nodeId = nodeId, response = response) + val pendingRequest = + PendingWearRequest( + nodeId = nodeId, + response = response, + preferredPhone = preferredPhone?.takeIf { it.nodeId == nodeId }, + ) check(pending.putIfAbsent(requestId, pendingRequest) == null) return try { try { @@ -136,14 +151,16 @@ internal class WearProxyClient private constructor( ) } catch (_: CancellationException) { currentCoroutineContext().ensureActive() + invalidatePreferredPhone(preferredPhone?.takeIf { it.nodeId == nodeId }) throw WearProxyException("phone_unavailable", "Paired phone is unavailable") } catch (_: Throwable) { + invalidatePreferredPhone(preferredPhone?.takeIf { it.nodeId == nodeId }) throw WearProxyException("phone_unavailable", "Paired phone is unavailable") } val envelope = response.await() if ( (expectedNodeId == null || requirePreferredNode || method.requiresPreferredSnapshotSource()) && - selectedPhoneNodeId.get() != nodeId + currentPreferredPhone()?.nodeId != nodeId ) { throw WearProxyException("phone_changed", "Preferred phone changed during request") } @@ -175,8 +192,12 @@ internal class WearProxyClient private constructor( path == WearProtocol.RESPONSE_PATH && message is WearMessage.Response -> { pending[message.requestId] ?.takeIf { it.nodeId == sourceNodeId } - ?.response - ?.complete(message) + ?.let { request -> + // Correlation is the reachability proof. Advance the registration + // before a concurrently expiring request can invalidate it. + confirmPreferredPhoneResponse(request.preferredPhone) + request.response.complete(message) + } null } path == WearProtocol.EVENT_PATH && message is WearMessage.Event -> { @@ -209,15 +230,12 @@ internal class WearProxyClient private constructor( } ?: throw WearProxyException("phone_unavailable", "Paired phone is unavailable") private suspend fun acceptEventSource(sourceNodeId: String): Boolean { - val snapshot = preferredPhoneSnapshot() - if (snapshot.known) { - val preferred = snapshot.nodeId - selectedPhoneNodeId.set(preferred) - return preferred == sourceNodeId + val preferredPhone = currentPreferredPhone() + if (preferredPhone != null) { + return preferredPhone.nodeId == sourceNodeId } - if (selectedPhoneNodeId.get() == sourceNodeId) return true return try { - resolvePreferredPhoneNode() == sourceNodeId + resolvePreferredPhone().nodeId == sourceNodeId } catch (err: CancellationException) { throw err } catch (_: WearProxyException) { @@ -225,49 +243,97 @@ internal class WearProxyClient private constructor( } } - /** Capability callbacks invalidate cached routing before another old-phone event is accepted. */ - fun updatePreferredPhoneNodeId(nodeId: String?) { + /** A unique directly connected phone becomes the preferred routing source immediately. */ + fun updatePreferredPhoneNodeId(nodeId: String) { val changed = synchronized(preferredPhoneLock) { - val changed = !preferredPhoneKnown || preferredPhoneNodeId != nodeId - preferredPhoneKnown = true - preferredPhoneNodeId = nodeId + val changed = registeredPhone?.nodeId != nodeId + preferredPhoneGeneration += 1 + registeredPhone = PreferredPhoneRegistration(nodeId, preferredPhoneGeneration) changed } - selectedPhoneNodeId.set(nodeId) if (changed) mutablePreferredPhoneChanges.tryEmit(nodeId) } - private suspend fun resolvePreferredPhoneNode(): String { - preferredPhoneSnapshot().takeIf(PreferredPhoneSnapshot::known)?.let { snapshot -> - return snapshot.nodeId ?: throw WearProxyException("phone_unavailable", "Paired phone is unavailable") - } + /** Capability callbacks are not reachability-filtered, so ambiguous results force fresh discovery. */ + fun invalidatePreferredPhoneNode() { + val changed = + synchronized(preferredPhoneLock) { + val changed = registeredPhone != null + preferredPhoneGeneration += 1 + registeredPhone = null + changed + } + if (changed) mutablePreferredPhoneChanges.tryEmit(null) + } + + private suspend fun resolvePreferredPhone(): PreferredPhoneRegistration { + // Capability callbacks can invalidate the route while discovery suspends. + // Only a result from the same generation may repopulate it. + val discoveryGeneration = + synchronized(preferredPhoneLock) { + registeredPhone?.let { return it } + preferredPhoneGeneration + } val resolved = resolvePhoneNode() - val selected = + return synchronized(preferredPhoneLock) { + registeredPhone ?: if (preferredPhoneGeneration == discoveryGeneration) { + preferredPhoneGeneration += 1 + PreferredPhoneRegistration(resolved, preferredPhoneGeneration).also { registeredPhone = it } + } else { + null + } + } ?: throw WearProxyException("phone_unavailable", "Paired phone is unavailable") + } + + private fun currentPreferredPhone(): PreferredPhoneRegistration? = + synchronized(preferredPhoneLock) { + registeredPhone + } + + private fun preferredPhoneRegistration(nodeId: String): PreferredPhoneRegistration? = + synchronized(preferredPhoneLock) { + registeredPhone?.takeIf { it.nodeId == nodeId } + } + + private fun invalidatePreferredPhone(registration: PreferredPhoneRegistration?) { + if (registration == null) return + val invalidated = synchronized(preferredPhoneLock) { - if (!preferredPhoneKnown) { - preferredPhoneKnown = true - preferredPhoneNodeId = resolved + if (registeredPhone == registration) { + // A capability callback can refresh the same node while an older request + // is failing. Clear only the registration that owned this transport attempt. + preferredPhoneGeneration += 1 + registeredPhone = null + true + } else { + false } - preferredPhoneNodeId - } ?: throw WearProxyException("phone_unavailable", "Paired phone is unavailable") - selectedPhoneNodeId.set(selected) - return selected + } + if (invalidated) mutablePreferredPhoneChanges.tryEmit(null) } - private fun preferredPhoneSnapshot(): PreferredPhoneSnapshot = + private fun confirmPreferredPhoneResponse(registration: PreferredPhoneRegistration?) { + if (registration == null) return synchronized(preferredPhoneLock) { - PreferredPhoneSnapshot(known = preferredPhoneKnown, nodeId = preferredPhoneNodeId) + if (registeredPhone == registration) { + // Any correlated response proves this registration is reachable. Advance + // it so an older parallel request cannot clear it on a later timeout. + preferredPhoneGeneration += 1 + registeredPhone = registration.copy(generation = preferredPhoneGeneration) + } } + } - private data class PreferredPhoneSnapshot( - val known: Boolean, - val nodeId: String?, + private data class PreferredPhoneRegistration( + val nodeId: String, + val generation: Long, ) private data class PendingWearRequest( val nodeId: String, val response: CompletableDeferred, + val preferredPhone: PreferredPhoneRegistration?, ) companion object { @@ -281,13 +347,13 @@ internal class WearProxyClient private constructor( return WearProxyClient( nodeResolver = WearNodeResolver { - capabilityClient - .getCapability(WearProtocol.PHONE_CAPABILITY, CapabilityClient.FILTER_REACHABLE) - .await() - .nodes - .sortedWith(compareByDescending { it.isNearby }.thenBy { it.id }) - .firstOrNull() - ?.id + selectReachablePhoneNodeId( + capabilityClient + .getCapability(WearProtocol.PHONE_CAPABILITY, CapabilityClient.FILTER_REACHABLE) + .await() + .nodes + .map { node -> WearReachablePhoneNode(id = node.id, isNearby = node.isNearby) }, + ) }, transport = WearMessageTransport { nodeId, path, data -> @@ -303,6 +369,22 @@ internal class WearProxyClient private constructor( } } +internal data class WearReachablePhoneNode( + val id: String, + val isNearby: Boolean, +) + +internal fun selectReachablePhoneNodeId(nodes: Collection): String? { + val distinctNodes = nodes.distinctBy(WearReachablePhoneNode::id) + val nearbyNodes = distinctNodes.filter(WearReachablePhoneNode::isNearby) + return when { + nearbyNodes.size == 1 -> nearbyNodes.single().id + nearbyNodes.isNotEmpty() -> null + distinctNodes.size == 1 -> distinctNodes.single().id + else -> null + } +} + private fun WearRpcMethod.requiresPreferredSnapshotSource(): Boolean = this == WearRpcMethod.ProxyStatus || this == WearRpcMethod.SessionsList || this == WearRpcMethod.ChatHistory internal enum class WearSequenceDecision { diff --git a/apps/android/wear/src/main/java/ai/openclaw/wear/WearProxyListenerService.kt b/apps/android/wear/src/main/java/ai/openclaw/wear/WearProxyListenerService.kt index efe10c706e1..2933b9c3e29 100644 --- a/apps/android/wear/src/main/java/ai/openclaw/wear/WearProxyListenerService.kt +++ b/apps/android/wear/src/main/java/ai/openclaw/wear/WearProxyListenerService.kt @@ -11,11 +11,19 @@ class WearProxyListenerService : WearableListenerService() { override fun onCapabilityChanged(capabilityInfo: CapabilityInfo) { if (capabilityInfo.name != WearProtocol.PHONE_CAPABILITY) return val preferredNodeId = - capabilityInfo.nodes - .sortedWith(compareByDescending { it.isNearby }.thenBy { it.id }) - .firstOrNull() - ?.id - (application as? WearApplication)?.proxyClient?.updatePreferredPhoneNodeId(preferredNodeId) + selectReachablePhoneNodeId( + capabilityInfo.nodes.map { node -> + WearReachablePhoneNode(id = node.id, isNearby = node.isNearby) + }, + ) + val proxyClient = (application as? WearApplication)?.proxyClient ?: return + if (preferredNodeId == null) { + // Capability callbacks contain reachable nodes. Multiple routes remain + // ambiguous, so force the next request through fresh discovery. + proxyClient.invalidatePreferredPhoneNode() + } else { + proxyClient.updatePreferredPhoneNodeId(preferredNodeId) + } } override fun onMessageReceived(messageEvent: MessageEvent) { diff --git a/apps/android/wear/src/test/java/ai/openclaw/wear/WearProxyClientTest.kt b/apps/android/wear/src/test/java/ai/openclaw/wear/WearProxyClientTest.kt index a23bc75f006..0dd00c0941b 100644 --- a/apps/android/wear/src/test/java/ai/openclaw/wear/WearProxyClientTest.kt +++ b/apps/android/wear/src/test/java/ai/openclaw/wear/WearProxyClientTest.kt @@ -8,13 +8,16 @@ import ai.openclaw.wear.shared.WearProtocolCodec import ai.openclaw.wear.shared.WearProxyCapability import ai.openclaw.wear.shared.WearRpcMethod import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.async import kotlinx.coroutines.awaitCancellation import kotlinx.coroutines.flow.first import kotlinx.coroutines.test.advanceUntilIdle import kotlinx.coroutines.test.runCurrent import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.withTimeout import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.jsonObject @@ -346,6 +349,374 @@ class WearProxyClientTest { assertEquals("phone_unavailable", (failure as WearProxyException).code) } + @Test + fun shorterCallerTimeoutPropagatesWithoutInvalidatingPhone() = + runTest { + var discoveries = 0 + var respond = false + lateinit var client: WearProxyClient + client = + WearProxyClient.createForTests( + nodeResolver = + WearNodeResolver { + discoveries += 1 + "resolver-phone" + }, + transport = + WearMessageTransport { nodeId, _, data -> + if (!respond) awaitCancellation() + val request = (WearProtocolCodec.decode(data) as WearDecodeResult.Success).message as WearMessage.Request + client.handleMessage( + sourceNodeId = nodeId, + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = request.requestId, ok = true)), + ) + }, + ) + client.updatePreferredPhoneNodeId("phone-current") + + val failure = + runCatching { + withTimeout(1_000L) { + client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) + } + }.exceptionOrNull() + respond = true + + assertTrue(failure is TimeoutCancellationException) + assertEquals("phone-current", client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null).sourceNodeId) + assertEquals(0, discoveries) + } + + @Test + fun sendFailureInvalidatesCachedPhoneAndRediscoversNextRequest() = + runTest { + var resolvedNode = "phone-old" + var discoveries = 0 + var sends = 0 + val sentNodes = mutableListOf() + lateinit var client: WearProxyClient + client = + WearProxyClient.createForTests( + nodeResolver = + WearNodeResolver { + discoveries += 1 + resolvedNode + }, + transport = + WearMessageTransport { nodeId, _, data -> + sends += 1 + sentNodes += nodeId + if (sends == 1) error("stale node") + val request = (WearProtocolCodec.decode(data) as WearDecodeResult.Success).message as WearMessage.Request + client.handleMessage( + sourceNodeId = nodeId, + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = request.requestId, ok = true)), + ) + }, + ) + + val first = runCatching { client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) }.exceptionOrNull() + resolvedNode = "phone-new" + val second = client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) + + assertEquals("phone_unavailable", (first as WearProxyException).code) + assertEquals("phone-new", second.sourceNodeId) + assertEquals(2, discoveries) + assertEquals(listOf("phone-old", "phone-new"), sentNodes) + } + + @Test + fun responseTimeoutInvalidatesCachedPhoneAndRediscoversNextRequest() = + runTest { + var resolvedNode = "phone-old" + var discoveries = 0 + var sends = 0 + lateinit var client: WearProxyClient + client = + WearProxyClient.createForTests( + nodeResolver = + WearNodeResolver { + discoveries += 1 + resolvedNode + }, + transport = + WearMessageTransport { nodeId, _, data -> + sends += 1 + if (sends > 1) { + val request = (WearProtocolCodec.decode(data) as WearDecodeResult.Success).message as WearMessage.Request + client.handleMessage( + sourceNodeId = nodeId, + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = request.requestId, ok = true)), + ) + } + }, + ) + + val first = runCatching { client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) }.exceptionOrNull() + resolvedNode = "phone-new" + val second = client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) + + assertEquals("timeout", (first as WearProxyException).code) + assertEquals("phone-new", second.sourceNodeId) + assertEquals(2, discoveries) + } + + @Test + fun staleSendFailureDoesNotInvalidateNewerSamePhoneGeneration() = + runTest { + var discoveries = 0 + var sends = 0 + lateinit var client: WearProxyClient + client = + WearProxyClient.createForTests( + nodeResolver = + WearNodeResolver { + discoveries += 1 + "resolver-phone" + }, + transport = + WearMessageTransport { nodeId, _, data -> + sends += 1 + if (sends == 1) { + client.updatePreferredPhoneNodeId(nodeId) + error("stale send") + } + val request = (WearProtocolCodec.decode(data) as WearDecodeResult.Success).message as WearMessage.Request + client.handleMessage( + sourceNodeId = nodeId, + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = request.requestId, ok = true)), + ) + }, + ) + client.updatePreferredPhoneNodeId("phone-current") + + val first = runCatching { client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) }.exceptionOrNull() + val second = client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) + + assertEquals("phone_unavailable", (first as WearProxyException).code) + assertEquals("phone-current", second.sourceNodeId) + assertEquals(0, discoveries) + } + + @Test + fun staleResponseTimeoutDoesNotInvalidateNewerSamePhoneGeneration() = + runTest { + var discoveries = 0 + var sends = 0 + lateinit var client: WearProxyClient + client = + WearProxyClient.createForTests( + nodeResolver = + WearNodeResolver { + discoveries += 1 + "resolver-phone" + }, + transport = + WearMessageTransport { nodeId, _, data -> + sends += 1 + if (sends == 1) { + client.updatePreferredPhoneNodeId(nodeId) + } else { + val request = (WearProtocolCodec.decode(data) as WearDecodeResult.Success).message as WearMessage.Request + client.handleMessage( + sourceNodeId = nodeId, + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = request.requestId, ok = true)), + ) + } + }, + ) + client.updatePreferredPhoneNodeId("phone-current") + + val first = runCatching { client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) }.exceptionOrNull() + val second = client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) + + assertEquals("timeout", (first as WearProxyException).code) + assertEquals("phone-current", second.sourceNodeId) + assertEquals(0, discoveries) + } + + @Test + fun successfulParallelResponseProtectsPhoneFromOlderTimeout() = + runTest { + var discoveries = 0 + var sends = 0 + lateinit var client: WearProxyClient + client = + WearProxyClient.createForTests( + nodeResolver = + WearNodeResolver { + discoveries += 1 + "resolver-phone" + }, + transport = + WearMessageTransport { nodeId, _, data -> + sends += 1 + if (sends > 1) { + val request = (WearProtocolCodec.decode(data) as WearDecodeResult.Success).message as WearMessage.Request + client.handleMessage( + sourceNodeId = nodeId, + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = request.requestId, ok = true)), + ) + } + }, + ) + client.updatePreferredPhoneNodeId("phone-current") + + val older = async { runCatching { client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) } } + runCurrent() + val newer = async { client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) } + runCurrent() + + assertEquals("phone-current", newer.await().sourceNodeId) + assertEquals("timeout", (older.await().exceptionOrNull() as WearProxyException).code) + assertEquals("phone-current", client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null).sourceNodeId) + assertEquals(0, discoveries) + } + + @Test + fun correlatedResponseProtectsPhoneBeforeRequesterResumes() = + runTest { + var discoveries = 0 + var respond = false + val requests = mutableListOf() + lateinit var client: WearProxyClient + client = + WearProxyClient.createForTests( + nodeResolver = + WearNodeResolver { + discoveries += 1 + "resolver-phone" + }, + transport = + WearMessageTransport { nodeId, _, data -> + val request = (WearProtocolCodec.decode(data) as WearDecodeResult.Success).message as WearMessage.Request + requests += request + if (respond) { + client.handleMessage( + sourceNodeId = nodeId, + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = request.requestId, ok = true)), + ) + } + }, + ) + client.updatePreferredPhoneNodeId("phone-current") + + val older = async { runCatching { client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) } } + val newer = async { runCatching { client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) } } + runCurrent() + assertEquals(2, requests.size) + + client.handleMessage( + sourceNodeId = "phone-current", + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = requests[1].requestId, ok = true)), + ) + newer.cancel() + advanceUntilIdle() + + assertTrue(newer.isCancelled) + assertEquals("timeout", (older.await().exceptionOrNull() as WearProxyException).code) + respond = true + assertEquals("phone-current", client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null).sourceNodeId) + assertEquals(0, discoveries) + } + + @Test + fun ambiguousCapabilityCallbackForcesReachableRediscovery() = + runTest { + var discoveries = 0 + lateinit var client: WearProxyClient + client = + WearProxyClient.createForTests( + nodeResolver = + WearNodeResolver { + discoveries += 1 + "phone-reachable" + }, + transport = + WearMessageTransport { nodeId, _, data -> + val request = (WearProtocolCodec.decode(data) as WearDecodeResult.Success).message as WearMessage.Request + client.handleMessage( + sourceNodeId = nodeId, + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = request.requestId, ok = true)), + ) + }, + ) + client.updatePreferredPhoneNodeId("phone-stale") + + client.invalidatePreferredPhoneNode() + val result = client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) + + assertEquals("phone-reachable", result.sourceNodeId) + assertEquals(1, discoveries) + } + + @Test + fun inFlightDiscoveryCannotRestoreInvalidatedPhone() = + runTest { + val discoveryStarted = CompletableDeferred() + val releaseDiscovery = CompletableDeferred() + var resolvedNode = "phone-old" + var discoveries = 0 + val sentNodes = mutableListOf() + lateinit var client: WearProxyClient + client = + WearProxyClient.createForTests( + nodeResolver = + WearNodeResolver { + val result = resolvedNode + discoveries += 1 + if (discoveries == 1) { + discoveryStarted.complete(Unit) + releaseDiscovery.await() + } + result + }, + transport = + WearMessageTransport { nodeId, _, data -> + sentNodes += nodeId + val request = (WearProtocolCodec.decode(data) as WearDecodeResult.Success).message as WearMessage.Request + client.handleMessage( + sourceNodeId = nodeId, + path = WearProtocol.RESPONSE_PATH, + data = WearProtocolCodec.encode(WearMessage.Response(requestId = request.requestId, ok = true)), + ) + }, + ) + + val first = async { runCatching { client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null) } } + discoveryStarted.await() + client.invalidatePreferredPhoneNode() + resolvedNode = "phone-new" + releaseDiscovery.complete(Unit) + + assertEquals("phone_unavailable", (first.await().exceptionOrNull() as WearProxyException).code) + assertEquals("phone-new", client.request(WearRpcMethod.ProxyStatus, buildJsonObject {}, null).sourceNodeId) + assertEquals(2, discoveries) + assertEquals(listOf("phone-new"), sentNodes) + } + + @Test + fun reachablePhoneSelectionPrefersOneNearbyNodeAndRejectsAmbiguity() { + val nearby = WearReachablePhoneNode(id = "phone-nearby", isNearby = true) + val nearbyTwo = WearReachablePhoneNode(id = "phone-nearby-2", isNearby = true) + val cloud = WearReachablePhoneNode(id = "phone-cloud", isNearby = false) + val cloudTwo = WearReachablePhoneNode(id = "phone-cloud-2", isNearby = false) + + assertEquals("phone-nearby", selectReachablePhoneNodeId(listOf(cloud, nearby))) + assertEquals("phone-cloud", selectReachablePhoneNodeId(listOf(cloud))) + assertEquals(null, selectReachablePhoneNodeId(listOf(nearby, nearbyTwo, cloud))) + assertEquals(null, selectReachablePhoneNodeId(listOf(cloud, cloudTwo))) + } + @Test fun eventGapAndResetRequireCanonicalRefresh() { val tracker = WearEventSequenceTracker() From 5360c75de23be2e53dc9c70d8da81c6f0580cef5 Mon Sep 17 00:00:00 2001 From: mushuiyu886 Date: Sat, 18 Jul 2026 18:34:21 +0800 Subject: [PATCH 33/76] fix(tui): stop Codex login lookup hanging the terminal (#109452) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(tui): bound Codex CLI lookup * fix(tui): bound Codex CLI lookup asynchronously Co-authored-by: 杨浩宇0668001029 * style(tui): format Codex auth flow Co-authored-by: 杨浩宇0668001029 --------- Co-authored-by: Peter Steinberger --- src/tui/tui.resolve-codex-bin.test.ts | 65 ++++++++++++------ src/tui/tui.test.ts | 8 +-- src/tui/tui.ts | 98 +++++++++++++++------------ 3 files changed, 103 insertions(+), 68 deletions(-) diff --git a/src/tui/tui.resolve-codex-bin.test.ts b/src/tui/tui.resolve-codex-bin.test.ts index 68115ee6cae..a3965d9f6d4 100644 --- a/src/tui/tui.resolve-codex-bin.test.ts +++ b/src/tui/tui.resolve-codex-bin.test.ts @@ -1,29 +1,50 @@ -// Covers TUI Codex CLI lookup command selection. +// Covers bounded TUI Codex CLI lookup command selection. import fs from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { withMockedWindowsPlatform } from "../test-utils/vitest-spies.js"; +import { withMockedPlatform, withMockedWindowsPlatform } from "../test-utils/vitest-spies.js"; -const execFileSyncMock = vi.hoisted(() => vi.fn()); +const runCommandWithTimeoutMock = vi.hoisted(() => vi.fn()); -vi.mock("node:child_process", async () => { - const actual = await vi.importActual("node:child_process"); - return { - ...actual, - execFileSync: execFileSyncMock, - }; -}); +vi.mock("../process/exec.js", () => ({ runCommandWithTimeout: runCommandWithTimeoutMock })); import { resolveCodexCliBin } from "./tui.js"; afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); - execFileSyncMock.mockReset(); + runCommandWithTimeoutMock.mockReset(); }); describe("resolveCodexCliBin", () => { - it("uses the trusted Windows where.exe when resolving codex", async () => { + it("bounds lookup and returns the first PATH match", async () => { + runCommandWithTimeoutMock.mockResolvedValue({ + code: 0, + stdout: "/usr/local/bin/codex\n/opt/bin/codex\n", + termination: "exit", + }); + + await withMockedPlatform("linux", async () => { + await expect(resolveCodexCliBin()).resolves.toBe("/usr/local/bin/codex"); + }); + expect(runCommandWithTimeoutMock).toHaveBeenCalledWith(["which", "codex"], { + killSignal: "SIGKILL", + maxOutputBytes: 64 * 1024, + timeoutMs: 5_000, + }); + }); + + it("returns null when lookup times out", async () => { + runCommandWithTimeoutMock.mockResolvedValue({ + code: null, + stdout: "", + termination: "timeout", + }); + + await expect(resolveCodexCliBin()).resolves.toBeNull(); + }); + + it("uses the trusted Windows where.exe", async () => { const accessSync = fs.accessSync.bind(fs); vi.spyOn(fs, "accessSync").mockImplementation((filePath, mode) => { if (String(filePath).toLowerCase() === "c:\\windows\\system32\\reg.exe") { @@ -32,16 +53,22 @@ describe("resolveCodexCliBin", () => { return accessSync(filePath, mode); }); vi.stubEnv("SystemRoot", "D:\\Windows"); - execFileSyncMock.mockReturnValue("D:\\Tools\\codex.exe\r\n"); + runCommandWithTimeoutMock.mockResolvedValue({ + code: 0, + stdout: "D:\\Tools\\codex.exe\r\n", + termination: "exit", + }); await withMockedWindowsPlatform(async () => { - expect(resolveCodexCliBin()).toBe("D:\\Tools\\codex.exe"); + await expect(resolveCodexCliBin()).resolves.toBe("D:\\Tools\\codex.exe"); }); - - expect(execFileSyncMock).toHaveBeenCalledWith( - path.win32.join("D:\\Windows", "System32", "where.exe"), - ["codex"], - { encoding: "utf8" }, + expect(runCommandWithTimeoutMock).toHaveBeenCalledWith( + [path.win32.join("D:\\Windows", "System32", "where.exe"), "codex"], + { + killSignal: "SIGKILL", + maxOutputBytes: 64 * 1024, + timeoutMs: 5_000, + }, ); }); }); diff --git a/src/tui/tui.test.ts b/src/tui/tui.test.ts index 1861a6921ae..39b4e46458d 100644 --- a/src/tui/tui.test.ts +++ b/src/tui/tui.test.ts @@ -583,8 +583,8 @@ describe("TUI shutdown safety", () => { }); describe("resolveCodexCliBin", () => { - it("returns a string path when codex CLI is installed", () => { - const result = resolveCodexCliBin(); + it("returns a string path when codex CLI is installed", async () => { + const result = await resolveCodexCliBin(); // In this test environment codex is installed; verify it returns a non-empty path if (result !== null) { expect(typeof result).toBe("string"); @@ -593,8 +593,8 @@ describe("resolveCodexCliBin", () => { } }); - it("returns null or a valid path (never throws)", () => { - const result = resolveCodexCliBin(); + it("returns null or a valid path (never throws)", async () => { + const result = await resolveCodexCliBin(); if (result === null) { expect(result).toBeNull(); } else { diff --git a/src/tui/tui.ts b/src/tui/tui.ts index 9b1d446fd8c..a8a618519ad 100644 --- a/src/tui/tui.ts +++ b/src/tui/tui.ts @@ -1,5 +1,5 @@ // Runs the interactive TUI loop and coordinates backend, input, and rendering. -import { execFileSync, spawn } from "node:child_process"; +import { spawn } from "node:child_process"; import { existsSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -23,6 +23,7 @@ import { registerUncaughtExceptionHandler } from "../infra/unhandled-rejections. import { getWindowsSystem32ExePath } from "../infra/windows-install-roots.js"; import { setConsoleSubsystemFilter } from "../logging/console.js"; import { loggingState } from "../logging/state.js"; +import { runCommandWithTimeout } from "../process/exec.js"; import { buildWindowsCmdExeCommandLine, isWindowsBatchCommand, @@ -102,6 +103,7 @@ const OPENCLAW_DIST_ENTRY_MJS_PATH = fileURLToPath( ); const OPENAI_CODEX_PROVIDER = "openai"; +const CODEX_CLI_LOOKUP_TIMEOUT_MS = 5_000; type RunTuiOptions = TuiOptions & { backend?: TuiBackend; @@ -117,13 +119,20 @@ type RunTuiOptions = TuiOptions & { }; /** Resolve the absolute path to the `codex` CLI binary, or `null` if not installed. */ -export function resolveCodexCliBin(): string | null { +export async function resolveCodexCliBin(): Promise { + const lookupCommand = + process.platform === "win32" ? getWindowsSystem32ExePath("where.exe") : "which"; try { - const lookupCmd = - process.platform === "win32" ? getWindowsSystem32ExePath("where.exe") : "which"; - // `where` on Windows can return multiple lines; take the first match. - const raw = execFileSync(lookupCmd, ["codex"], { encoding: "utf8" }).trim(); - return raw.split(/\r?\n/)[0] || null; + const result = await runCommandWithTimeout([lookupCommand, "codex"], { + killSignal: "SIGKILL", + maxOutputBytes: 64 * 1024, + timeoutMs: CODEX_CLI_LOOKUP_TIMEOUT_MS, + }); + if (result.code !== 0 || result.termination !== "exit") { + return null; + } + // `where` on Windows can return multiple matches; use PATH order. + return result.stdout.trim().split(/\r?\n/)[0]?.trim() || null; } catch { return null; } @@ -1183,45 +1192,44 @@ export async function runTui(opts: RunTuiOptions): Promise { const runAuthFlow = isLocalMode ? async (params: { provider?: string }) => - await withTuiSuspended( - async () => - await new Promise<{ exitCode: number | null; signal: NodeJS.Signals | null }>( - (resolve, reject) => { - const provider = params.provider?.trim() || undefined; - - // Codex owns its auth store; delegate when the CLI is available. - const codexBin = - provider === OPENAI_CODEX_PROVIDER || - (!provider && sessionInfo.modelProvider === OPENAI_CODEX_PROVIDER) - ? resolveCodexCliBin() - : null; - - let command: string; - let args: string[]; - if (codexBin) { - command = codexBin; - args = ["login"]; - } else { - ({ command, args } = resolveLocalAuthCliInvocation()); - if (provider) { - args.push("--provider", provider); - } + await withTuiSuspended(async () => { + const provider = params.provider?.trim() || undefined; + + // Codex owns its auth store; delegate when the CLI is available. + const codexBin = + provider === OPENAI_CODEX_PROVIDER || + (!provider && sessionInfo.modelProvider === OPENAI_CODEX_PROVIDER) + ? await resolveCodexCliBin() + : null; + + return await new Promise<{ exitCode: number | null; signal: NodeJS.Signals | null }>( + (resolve, reject) => { + let command: string; + let args: string[]; + if (codexBin) { + command = codexBin; + args = ["login"]; + } else { + ({ command, args } = resolveLocalAuthCliInvocation()); + if (provider) { + args.push("--provider", provider); } - - const invocation = resolveLocalAuthSpawnInvocation({ command, args }); - const child = spawn(invocation.command, invocation.args, { - cwd: resolveLocalAuthSpawnCwd({ args, defaultCwd: resolveUsableCwd() }), - env: process.env, - stdio: "inherit", - ...invocation.options, - }); - child.once("error", reject); - child.once("exit", (exitCode, signal) => { - resolve({ exitCode, signal }); - }); - }, - ), - ) + } + + const invocation = resolveLocalAuthSpawnInvocation({ command, args }); + const child = spawn(invocation.command, invocation.args, { + cwd: resolveLocalAuthSpawnCwd({ args, defaultCwd: resolveUsableCwd() }), + env: process.env, + stdio: "inherit", + ...invocation.options, + }); + child.once("error", reject); + child.once("exit", (exitCode, signal) => { + resolve({ exitCode, signal }); + }); + }, + ); + }) : undefined; const updateFooter = () => { From bc6d9eb16a2243d983e4bd1a3cec5600fba873ab Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 11:39:01 +0100 Subject: [PATCH 34/76] fix: custom Anthropic models work without maxTokens (#110617) * fix(agents): default Anthropic max output tokens * fix(agents): bound omitted Anthropic output cap --- src/agents/anthropic-transport-stream.test.ts | 84 +++++++++++++++++-- src/agents/anthropic-transport-stream.ts | 36 ++++++-- 2 files changed, 104 insertions(+), 16 deletions(-) diff --git a/src/agents/anthropic-transport-stream.test.ts b/src/agents/anthropic-transport-stream.test.ts index 90eac8289b8..6fb5429102e 100644 --- a/src/agents/anthropic-transport-stream.test.ts +++ b/src/agents/anthropic-transport-stream.test.ts @@ -1262,19 +1262,19 @@ describe("anthropic transport stream", () => { expect(latestAnthropicRequest().payload.stream).toBe(true); }); - it("fails locally when Anthropic maxTokens is non-positive after resolution", async () => { + it("uses a default maxTokens for custom Anthropic models that omit it", async () => { const model = attachModelProviderRequestTransport( { - id: "claude-haiku-4-5", - name: "Claude Haiku 4.5", + id: "custom-model", + name: "Custom Model", api: "anthropic-messages", - provider: "anthropic", - baseUrl: "https://api.anthropic.com", + provider: "custom-anthropic", + baseUrl: "https://custom.example/anthropic", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 32000, - maxTokens: 0, + contextWindow: 200_000, + maxTokens: undefined as never, } satisfies Model<"anthropic-messages">, { proxy: { @@ -1298,6 +1298,76 @@ describe("anthropic transport stream", () => { const result = await stream.result(); + expect(result.stopReason).toBe("stop"); + expect(latestAnthropicRequest().payload).toMatchObject({ + model: "custom-model", + max_tokens: 4_096, + stream: true, + }); + }); + + it("clamps the custom Anthropic maxTokens fallback to the context window", async () => { + const model = attachModelProviderRequestTransport( + { + id: "custom-model", + name: "Custom Model", + api: "anthropic-messages", + provider: "custom-anthropic", + baseUrl: "https://custom.example/anthropic", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 4_096, + maxTokens: undefined as never, + } satisfies Model<"anthropic-messages">, + { + proxy: { + mode: "env-proxy", + }, + }, + ); + + await runTransportStream( + model, + { messages: [{ role: "user", content: "hello" }] } as AnthropicStreamContext, + { apiKey: "fake" } as AnthropicStreamOptions, + ); + + expect(latestAnthropicRequest().payload.max_tokens).toBe(1_024); + }); + + it("fails locally when a custom Anthropic model has an invalid maxTokens", async () => { + const model = attachModelProviderRequestTransport( + { + id: "custom-model", + name: "Custom Model", + api: "anthropic-messages", + provider: "custom-anthropic", + baseUrl: "https://custom.example/anthropic", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32_000, + maxTokens: 0, + } satisfies Model<"anthropic-messages">, + { + proxy: { + mode: "env-proxy", + }, + }, + ); + const streamFn = createAnthropicMessagesTransportStreamFn(); + + const stream = await Promise.resolve( + streamFn( + model, + { messages: [{ role: "user", content: "hello" }] } as Parameters[1], + { apiKey: "fake" } as Parameters[2], + ), + ); + + const result = await stream.result(); + expect(result.stopReason).toBe("error"); expect(result.errorMessage).toContain( "Anthropic Messages transport requires a positive maxTokens value", diff --git a/src/agents/anthropic-transport-stream.ts b/src/agents/anthropic-transport-stream.ts index bf5f21532f5..e17dac85bfb 100644 --- a/src/agents/anthropic-transport-stream.ts +++ b/src/agents/anthropic-transport-stream.ts @@ -94,6 +94,8 @@ const CLAUDE_CODE_BILLING_SYSTEM_BLOCK = `x-anthropic-billing-header: cc_version const ANTHROPIC_MESSAGES_ERROR_BODY_MAX_BYTES = 8 * 1024; const ANTHROPIC_MESSAGES_ERROR_BODY_MAX_CHARS = 400; const ANTHROPIC_MESSAGES_ERROR_BODY_READ_IDLE_TIMEOUT_MS = 10_000; +const ANTHROPIC_MESSAGES_DEFAULT_MAX_TOKENS = 4_096; +const ANTHROPIC_MESSAGES_FALLBACK_CONTEXT_DIVISOR = 4; // Mirror the fetch sanitizer cap here because compatible routes such as Kimi // bypass that layer; without a parser-local guard, partial frames grow forever. const ANTHROPIC_MESSAGES_SSE_PENDING_BUFFER_MAX_CHARS = 16 * 1024 * 1024; @@ -242,7 +244,7 @@ function clampReasoningLevel(level: ThinkingLevel): "minimal" | "low" | "medium" return level === "xhigh" || level === "max" ? "high" : level; } -function resolvePositiveAnthropicMaxTokens(value: unknown): number | undefined { +function resolvePositiveAnthropicTokenLimit(value: unknown): number | undefined { if (typeof value !== "number" || !Number.isFinite(value)) { return undefined; } @@ -251,20 +253,34 @@ function resolvePositiveAnthropicMaxTokens(value: unknown): number | undefined { } function resolveAnthropicMessagesMaxTokens(params: { + modelContextWindow: number | undefined; modelMaxTokens: number | undefined; requestedMaxTokens: number | undefined; useModelDefault?: boolean; }): number | undefined { - const requested = resolvePositiveAnthropicMaxTokens(params.requestedMaxTokens); + const requested = resolvePositiveAnthropicTokenLimit(params.requestedMaxTokens); if (requested !== undefined) { return requested; } - const modelMax = resolvePositiveAnthropicMaxTokens(params.modelMaxTokens); - return modelMax !== undefined - ? params.useModelDefault - ? modelMax - : Math.min(modelMax, 32_000) - : undefined; + const modelMax = resolvePositiveAnthropicTokenLimit(params.modelMaxTokens); + if (modelMax !== undefined) { + return params.useModelDefault ? modelMax : Math.min(modelMax, 32_000); + } + if (params.modelMaxTokens !== undefined) { + return undefined; + } + // Anthropic requires max_tokens even when an optional custom-model row has no output cap. + // Use a conservative compatibility baseline; higher model limits require explicit metadata. + const contextWindow = resolvePositiveAnthropicTokenLimit(params.modelContextWindow); + return contextWindow === undefined + ? ANTHROPIC_MESSAGES_DEFAULT_MAX_TOKENS + : Math.max( + 1, + Math.min( + ANTHROPIC_MESSAGES_DEFAULT_MAX_TOKENS, + Math.floor(contextWindow / ANTHROPIC_MESSAGES_FALLBACK_CONTEXT_DIVISOR), + ), + ); } function adjustMaxTokensForThinking(params: { @@ -1000,6 +1016,7 @@ function buildAnthropicParams( const mandatoryAdaptiveThinking = requiresClaudeAdaptiveThinking(model); const replayThinkingEnabled = mandatoryAdaptiveThinking || options?.thinkingEnabled === true; const maxTokens = resolveAnthropicMessagesMaxTokens({ + modelContextWindow: model.contextWindow, modelMaxTokens: model.maxTokens, requestedMaxTokens: options?.maxTokens, }); @@ -1130,6 +1147,7 @@ function resolveAnthropicTransportOptions( apiKey: string, ): AnthropicTransportOptions { const baseMaxTokens = resolveAnthropicMessagesMaxTokens({ + modelContextWindow: model.contextWindow, modelMaxTokens: model.maxTokens, requestedMaxTokens: options?.maxTokens, useModelDefault: resolveClaudeSonnet5ModelIdentity(model) !== undefined, @@ -1140,7 +1158,7 @@ function resolveAnthropicTransportOptions( ); } const reasoningModelMaxTokens = - resolvePositiveAnthropicMaxTokens(model.maxTokens) ?? baseMaxTokens; + resolvePositiveAnthropicTokenLimit(model.maxTokens) ?? baseMaxTokens; const mandatoryAdaptiveThinking = requiresClaudeAdaptiveThinking(model); const reasoning = options?.reasoning === "off" && mandatoryAdaptiveThinking ? "low" : options?.reasoning; From 9d97e10efe08c3b5e3043da19b80883e933e26be Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 11:42:14 +0100 Subject: [PATCH 35/76] refactor: move non-session runtime journals to SQLite (#109427) * refactor: migrate runtime JSONL state to SQLite * chore: refresh SQLite migration validation * fix: preserve safe audit migration order * fix: sanitize retired audit archives safely * fix: satisfy SQLite storage architecture gates * fix: serialize plugin doctor state migrations * fix: bound memory event SQLite projections * fix: rotate memory event projections safely * fix: preserve imported event retention age * fix: harden memory event export projection * fix: canonicalize memory event migrations * fix: harden legacy journal migrations * fix: report unsafe legacy journal paths * fix: preserve journal ownership during migration * fix: keep legacy file imports doctor-only * fix: align SQLite audit CI coverage * refactor: keep state helpers module-private * fix: harden legacy state migration recovery * fix: reconcile SQLite state audit migration * test: use neutral audit redaction fixtures * fix: close SQLite migration recovery gaps * refactor: split audit migration recovery helpers * test: prove completed audit pads stay out of backups * fix: preserve audit record ordinals across blank lines * fix: align SQLite audit CI contracts * build: package SQLite audit E2E entries * fix: preserve audit ordinals in backup snapshots --- .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/tools/acp-agents.md | 7 +- .../memory-core/doctor-contract-api.test.ts | 668 ++++++++++- extensions/memory-core/doctor-contract-api.ts | 773 +++++++++++- .../memory-core/src/memory-events.test.ts | 24 +- .../memory-core/src/public-artifacts.test.ts | 44 +- extensions/memory-wiki/src/bridge.test.ts | 15 +- .../memory-wiki/src/source-page-shared.ts | 3 +- package.json | 2 +- .../check-database-first-legacy-stores.mjs | 2 +- scripts/plugin-sdk-surface-report.mjs | 11 +- ...parent-stream-store.sqlite.test-support.ts | 25 + .../acp-parent-stream-store.sqlite.test.ts | 105 ++ src/agents/acp-parent-stream-store.sqlite.ts | 77 ++ src/agents/acp-spawn-parent-stream.test.ts | 111 +- src/agents/acp-spawn-parent-stream.ts | 187 +-- src/agents/acp-spawn.test.ts | 27 +- src/agents/acp-spawn.ts | 18 +- .../gateway-cli/run.option-collisions.test.ts | 10 +- src/cli/gateway-cli/run.ts | 18 +- src/commands/doctor-config-audit-scrub.ts | 4 +- src/commands/doctor.e2e-harness.ts | 4 + src/config/io.audit.test-support.ts | 23 + src/config/io.audit.test.ts | 132 ++- src/config/io.audit.ts | 216 +++- src/config/io.observe-recovery.test.ts | 17 +- src/config/io.observe-recovery.ts | 1 - src/config/io.observe.ts | 2 - src/config/io.write.ts | 1 - src/infra/backup-create.test.ts | 253 ++++ src/infra/backup-create.ts | 69 +- src/infra/backup-volatile-filter.test.ts | 7 + src/infra/fs-safe-advanced.ts | 1 + src/infra/regular-file.ts | 1 - src/infra/sqlite-audit-record-store.test.ts | 120 ++ src/infra/sqlite-audit-record-store.ts | 222 ++++ .../state-migrations.audit-backup.test.ts | 242 ++++ src/infra/state-migrations.audit-backup.ts | 228 ++++ .../state-migrations.audit-checkpoints.ts | 273 +++++ .../state-migrations.audit-coordination.ts | 25 + src/infra/state-migrations.audit-logs.test.ts | 1042 +++++++++++++++++ src/infra/state-migrations.audit-logs.ts | 669 +++++++++++ .../state-migrations.audit-logs.types.ts | 14 + src/infra/state-migrations.audit-records.ts | 90 ++ ...tate-migrations.audit-recovery-protocol.ts | 161 +++ .../state-migrations.audit-recovery.test.ts | 542 +++++++++ src/infra/state-migrations.audit-recovery.ts | 710 +++++++++++ src/infra/state-migrations.audit-sanitized.ts | 72 ++ src/infra/state-migrations.doctor.ts | 151 ++- src/infra/state-migrations.test.ts | 99 ++ src/infra/state-migrations.types.ts | 3 + src/memory-host-sdk/event-export.ts | 20 + src/memory-host-sdk/event-store.ts | 359 ++++++ src/memory-host-sdk/event-types.ts | 80 ++ src/memory-host-sdk/events.ts | 165 +-- src/plugin-sdk/memory-core-host-events.ts | 17 +- src/plugin-sdk/memory-host-core.test.ts | 891 +++++++++++++- src/plugin-sdk/memory-host-core.ts | 429 ++++++- .../memory-host-event-export.test.ts | 223 ++++ src/plugin-sdk/memory-host-event-export.ts | 268 +++++ src/plugin-sdk/memory-host-events.test.ts | 313 +++-- src/plugin-sdk/memory-host-events.ts | 3 +- src/plugin-sdk/plugin-state-test-runtime.ts | 3 + src/plugin-state/plugin-state-store.sqlite.ts | 250 +++- src/plugin-state/plugin-state-store.test.ts | 93 ++ src/plugin-state/plugin-state-store.ts | 124 ++ src/plugins/doctor-contract-registry.ts | 10 + src/state/openclaw-agent-db.generated.d.ts | 9 + src/state/openclaw-agent-db.test.ts | 49 +- src/state/openclaw-agent-db.ts | 3 +- src/state/openclaw-agent-schema.generated.ts | 13 + src/state/openclaw-agent-schema.sql | 13 + src/state/openclaw-state-db.generated.d.ts | 1 + src/state/openclaw-state-db.test.ts | 50 + src/state/openclaw-state-db.ts | 28 + src/state/openclaw-state-lease.ts | 11 +- src/state/openclaw-state-schema.generated.ts | 5 +- src/state/openclaw-state-schema.sql | 5 +- src/system-agent/audit.test-support.ts | 14 + src/system-agent/audit.test.ts | 24 +- src/system-agent/audit.ts | 51 +- src/system-agent/operations-execute.ts | 4 +- src/system-agent/operations.setup.test.ts | 18 +- src/system-agent/operations.test.ts | 22 +- src/system-agent/rescue-channel.live.test.ts | 13 +- src/system-agent/rescue-message.test.ts | 20 +- .../system-agent-first-run-docker-client.ts | 17 +- ...check-database-first-legacy-stores.test.ts | 17 + test/scripts/test-projects.test.ts | 2 +- tsdown.config.ts | 2 + 90 files changed, 10449 insertions(+), 715 deletions(-) create mode 100644 src/agents/acp-parent-stream-store.sqlite.test-support.ts create mode 100644 src/agents/acp-parent-stream-store.sqlite.test.ts create mode 100644 src/agents/acp-parent-stream-store.sqlite.ts create mode 100644 src/config/io.audit.test-support.ts create mode 100644 src/infra/sqlite-audit-record-store.test.ts create mode 100644 src/infra/sqlite-audit-record-store.ts create mode 100644 src/infra/state-migrations.audit-backup.test.ts create mode 100644 src/infra/state-migrations.audit-backup.ts create mode 100644 src/infra/state-migrations.audit-checkpoints.ts create mode 100644 src/infra/state-migrations.audit-coordination.ts create mode 100644 src/infra/state-migrations.audit-logs.test.ts create mode 100644 src/infra/state-migrations.audit-logs.ts create mode 100644 src/infra/state-migrations.audit-logs.types.ts create mode 100644 src/infra/state-migrations.audit-records.ts create mode 100644 src/infra/state-migrations.audit-recovery-protocol.ts create mode 100644 src/infra/state-migrations.audit-recovery.test.ts create mode 100644 src/infra/state-migrations.audit-recovery.ts create mode 100644 src/infra/state-migrations.audit-sanitized.ts create mode 100644 src/memory-host-sdk/event-export.ts create mode 100644 src/memory-host-sdk/event-store.ts create mode 100644 src/memory-host-sdk/event-types.ts create mode 100644 src/plugin-sdk/memory-host-event-export.test.ts create mode 100644 src/plugin-sdk/memory-host-event-export.ts create mode 100644 src/system-agent/audit.test-support.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 001cb093102..10596e14b98 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -166,7 +166,7 @@ ae67c256150b14f1a58738dce0a703071035b86ec26fc344ab03e942253aa72c module/memory- 4205ab767b79e740c7c7137079b19b315c85f7f8c748592840f00bbf70a71709 module/memory-core-host-engine-foundation cb8dcef652580ca82242d6e9506eda094006c85848f9b16b8b1c1aadff1706a8 module/memory-core-host-engine-qmd bbc06f2bb23d9e4a3445744711f691de9f902270e61c9740fd746f222d10fccb module/memory-core-host-engine-storage -b3255578cdf355d224d840a57e4ee7e190e60c02cbed1bfff2a222c08c3df3a8 module/memory-core-host-events +39e86819441281365dedb3d7d59979672bb90196f49206edb50b879bd747c9f0 module/memory-core-host-events fe87e7d794bbbdaed3c834b218a787e98c0e69b26b0fac582b0668b8effab9b4 module/memory-core-host-multimodal 746e155bb06b398749a6a1bfdc1b435b1ec1941dca3854554a912577731218dd module/memory-core-host-query e2377479799d52d8f67dcccf8f499f28a63948b0c03fa6043fc67e476a22ec0d module/memory-core-host-runtime-cli @@ -175,7 +175,7 @@ f8eb835f0769eba8324033141ecddb9f45bdb3d3241d2a5959fe8426ba6a2280 module/memory- ebbf937b2258487f773b8a6fbd6c648feea96399cae5fbea18ec7d29e027523d module/memory-core-host-secret 16bdd82751898ffa98157fd27c86fa0eb9ea415fb8ed703eb4e9b170f0a9501b module/memory-core-host-status 646773d8282a2ac6a89101685c406200935ddb3459a2830e820da132fa433b3c module/memory-host-core -7d68cbdfa0d5fad1ab32ba3792787daf386bc4ff7975fd5e25117e92563a2897 module/memory-host-events +f7bde3a0231ac75c128d6ae367d5020de886715fe9edbb2faaf673c6bd168398 module/memory-host-events 059856bc3671ccba7a2191f47c0295e0a91a374f7048cc2bc22784b389d15182 module/memory-host-files ba07f497aa5469915ebb5b6787148ff1f0f267d7e48c48f0c95d32b7dda8d91a module/memory-host-markdown 896d315fb78af09effd0af467e2b6e84331272b0123d4132e5b1f2254e80108e module/memory-host-search diff --git a/docs/tools/acp-agents.md b/docs/tools/acp-agents.md index 3574190bb44..e55a0e909c2 100644 --- a/docs/tools/acp-agents.md +++ b/docs/tools/acp-agents.md @@ -548,10 +548,9 @@ Two ways to start an ACP session: `"parent"` streams initial ACP run progress summaries back to the requester - session as system events. Accepted responses include `streamLogPath` - pointing to a session-scoped JSONL log (`.acp-stream.jsonl`) you - can tail for full relay history. Parent progress streams show assistant - commentary and ACP status progress by default unless + session as system events. OpenClaw records the full relay history in the + child agent's SQLite state and removes it with the child session. Parent + progress streams show assistant commentary and ACP status progress by default unless `streaming.progress.commentary=false`. Discord also defaults parent previews to progress mode when no stream mode is configured. Status progress still honors `acp.stream.tagVisibility`, so tags such as `plan` diff --git a/extensions/memory-core/doctor-contract-api.test.ts b/extensions/memory-core/doctor-contract-api.test.ts index ffea3fcefaf..e0f60fbcc60 100644 --- a/extensions/memory-core/doctor-contract-api.test.ts +++ b/extensions/memory-core/doctor-contract-api.test.ts @@ -3,14 +3,16 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { DatabaseSync } from "node:sqlite"; -import { expectDefined } from "@openclaw/normalization-core"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { ensureMemoryIndexSchema, loadSqliteVecExtension, } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { readMemoryHostEventRecords } from "openclaw/plugin-sdk/memory-host-events"; import { createPluginStateKeyedStoreForTests, + getPluginStateCapacityForTests, + importPluginStateEntriesForDoctorForTests, resetPluginStateStoreForTests, } from "openclaw/plugin-sdk/plugin-state-test-runtime"; import type { @@ -32,12 +34,18 @@ import { shortTermTestState as shortTermTesting, } from "./src/test-helpers.js"; -function requireStateMigration(index: number) { - return expectDefined(stateMigrations[index], `Memory Core state migration ${index}`); -} - function createDoctorContext(env: NodeJS.ProcessEnv): PluginDoctorStateMigrationContext { return { + getPluginStateCapacity() { + return getPluginStateCapacityForTests("memory-core", env); + }, + importPluginStateEntries(options, entries) { + importPluginStateEntriesForDoctorForTests( + "memory-core", + { ...options, env: options.env ?? env }, + entries, + ); + }, openPluginStateKeyedStore(options: OpenKeyedStoreOptions) { return createPluginStateKeyedStoreForTests("memory-core", { ...options, @@ -57,6 +65,26 @@ function legacyMemoryIndexMigration() { return migration; } +function dreamingStateMigration() { + const migration = stateMigrations.find( + (entry) => entry.id === "memory-core-dreams-json-to-sqlite", + ); + if (!migration) { + throw new Error("expected memory-core dreaming state migration"); + } + return migration; +} + +function hostEventsMigration() { + const migration = stateMigrations.find( + (entry) => entry.id === "memory-core-host-events-jsonl-to-sqlite", + ); + if (!migration) { + throw new Error("expected memory-core host events migration"); + } + return migration; +} + function qmdFileLockMigration() { const migration = stateMigrations.find( (entry) => entry.id === "memory-core-qmd-file-locks-to-sqlite-leases", @@ -458,6 +486,628 @@ describe("memory-core doctor dreaming migration", () => { }; } + it("treats a missing legacy host event directory as no state", async () => { + await fs.rm(path.join(workspaceDir, "memory"), { recursive: true }); + const migration = hostEventsMigration(); + + await expect(migration.detectLegacyState(migrationParams())).resolves.toBeNull(); + await expect(migration.migrateLegacyState(migrationParams())).resolves.toEqual({ + changes: [], + warnings: [], + }); + }); + + it("imports legacy memory host events into plugin state", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + await fs.writeFile( + eventPath, + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "sqlite policy", + resultCount: 0, + results: [], + })}\n`, + "utf8", + ); + const migration = hostEventsMigration(); + await expect(migration.detectLegacyState(migrationParams())).resolves.toEqual({ + preview: [expect.stringContaining("Memory Core host events")], + }); + const store = context().openPluginStateKeyedStore<{ + kind: "event"; + workspaceKey: string; + event: { type: string; query: string }; + recordedAt: number; + sequence: number; + }>({ namespace: "memory-host.events", maxEntries: 10_000 }); + await store.register("runtime-event", { + kind: "event", + workspaceKey: path.resolve(workspaceDir).replace(/\\/g, "/"), + event: { + type: "memory.recall.recorded", + query: "runtime after upgrade", + }, + recordedAt: Date.parse("2026-07-02T00:00:00.000Z"), + sequence: 1, + }); + const result = await migration.migrateLegacyState(migrationParams()); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated Memory Core host events -> SQLite plugin state (1 new row(s))", + expect.stringContaining("Archived Memory Core host events legacy source"), + ]); + const entries = await store.entries(); + const events = entries + .flatMap((entry) => (entry.value.kind === "event" ? [entry.value] : [])) + .toSorted((left, right) => left.sequence - right.sequence); + expect(events.map((entry) => entry.event.query)).toEqual([ + "sqlite policy", + "runtime after upgrade", + ]); + expect(events[0]?.sequence).toBeLessThan(0); + const migratedEntry = entries.find((entry) => entry.value.sequence < 0); + expect(migratedEntry?.createdAt).toBe(migratedEntry?.value.sequence); + const cursors = await context() + .openPluginStateKeyedStore<{ kind: "cursor"; lastSequence: number }>({ + namespace: "memory-host.event-cursors", + maxEntries: 1_000, + }) + .entries(); + expect(cursors).toHaveLength(1); + expect(cursors[0]?.value).toEqual({ kind: "cursor", lastSequence: 1 }); + await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined(); + }); + + it("keeps identical events recreated by an older writer as a new archive generation", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const event = { + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "repeated after downgrade", + resultCount: 0, + results: [], + }; + const migration = hostEventsMigration(); + await fs.writeFile(eventPath, `${JSON.stringify(event)}\n`, "utf8"); + await migration.migrateLegacyState(migrationParams()); + await fs.writeFile(eventPath, `${JSON.stringify(event)}\n`, "utf8"); + + const repeated = await migration.migrateLegacyState(migrationParams()); + + expect(repeated.warnings).toEqual([]); + expect(repeated.changes).toEqual([ + "Migrated Memory Core host events -> SQLite plugin state (1 new row(s))", + expect.stringContaining("events.jsonl.migrated.2"), + ]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toHaveLength(2); + await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined(); + await expect(fs.access(`${eventPath}.migrated.2`)).resolves.toBeUndefined(); + }); + + it("recovers appends written through an open legacy descriptor after archival", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const event = (query: string) => ({ + type: "memory.recall.recorded" as const, + timestamp: "2026-07-01T00:00:00.000Z", + query, + resultCount: 0, + results: [], + }); + await fs.writeFile(eventPath, `${JSON.stringify(event("before claim"))}\n`, "utf8"); + const oldWriter = await fs.open(eventPath, "a"); + const migration = hostEventsMigration(); + try { + await migration.migrateLegacyState(migrationParams()); + await fs.writeFile(eventPath, `${JSON.stringify(event("newer generation"))}\n`, "utf8"); + await migration.migrateLegacyState(migrationParams()); + await oldWriter.appendFile(`${JSON.stringify(event("late append"))}\n`, "utf8"); + await oldWriter.sync(); + } finally { + await oldWriter.close(); + } + + await expect(migration.detectLegacyState(migrationParams())).resolves.toEqual({ + preview: [expect.stringContaining("events.jsonl.migrated")], + }); + const recovered = await migration.migrateLegacyState(migrationParams()); + + expect(recovered.warnings).toEqual([]); + expect(recovered.changes).toEqual([ + expect.stringContaining("Recovered 1 later Memory Core host event row"), + ]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([ + { query: "before claim" }, + { query: "newer generation" }, + { query: "late append" }, + ]); + await expect( + readMemoryHostEventRecords({ workspaceDir, env, limit: 1 }), + ).resolves.toMatchObject([{ query: "late append" }]); + await expect(migration.detectLegacyState(migrationParams())).resolves.toBeNull(); + + await fs.writeFile(eventPath, `${JSON.stringify(event("after recovery generation"))}\n`); + await migration.migrateLegacyState(migrationParams()); + await expect( + readMemoryHostEventRecords({ workspaceDir, env, limit: 1 }), + ).resolves.toMatchObject([{ query: "after recovery generation" }]); + }); + + it("fails closed when a checkpointed host event archive changes other than by append", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const archivedPath = `${eventPath}.migrated`; + await fs.writeFile( + eventPath, + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "original archive row", + resultCount: 0, + results: [], + })}\n`, + "utf8", + ); + const migration = hostEventsMigration(); + await migration.migrateLegacyState(migrationParams()); + await fs.writeFile( + archivedPath, + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "rewritten archive row", + resultCount: 0, + results: [], + })}\n`, + "utf8", + ); + + const result = await migration.migrateLegacyState(migrationParams()); + + expect(result.changes).toEqual([]); + expect(result.warnings).toEqual([expect.stringContaining("changed other than by append")]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([ + { query: "original archive row" }, + ]); + await expect(fs.readFile(archivedPath, "utf8")).resolves.toContain("rewritten archive row"); + }); + + it("orders and limits migrated host events by archive generation", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const event = (query: string) => ({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query, + resultCount: 0, + results: [], + }); + const migration = hostEventsMigration(); + await fs.writeFile(eventPath, `${JSON.stringify(event("older generation"))}\n`, "utf8"); + await migration.migrateLegacyState(migrationParams()); + await fs.writeFile(eventPath, `${JSON.stringify(event("newer generation"))}\n`, "utf8"); + await migration.migrateLegacyState(migrationParams()); + + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([ + { query: "older generation" }, + { query: "newer generation" }, + ]); + await expect( + readMemoryHostEventRecords({ workspaceDir, env, limit: 1 }), + ).resolves.toMatchObject([{ query: "newer generation" }]); + }); + + it("refuses to replay a checkpointless older archive after a newer generation", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const event = (query: string) => ({ + type: "memory.recall.recorded" as const, + timestamp: "2026-07-01T00:00:00.000Z", + query, + resultCount: 0, + results: [], + }); + const migration = hostEventsMigration(); + await fs.writeFile(eventPath, `${JSON.stringify(event("older generation"))}\n`, "utf8"); + await migration.migrateLegacyState(migrationParams()); + await context() + .openPluginStateKeyedStore({ namespace: "memory-host.events", maxEntries: 10_000 }) + .clear(); + await fs.writeFile(eventPath, `${JSON.stringify(event("newer generation"))}\n`, "utf8"); + await migration.migrateLegacyState(migrationParams()); + await context() + .openPluginStateKeyedStore({ + namespace: "memory-host.event-migration-checkpoints", + maxEntries: 10_000, + overflowPolicy: "reject-new", + }) + .clear(); + + const replay = await migration.migrateLegacyState(migrationParams()); + + expect(replay.changes).toEqual([]); + expect(replay.warnings).toEqual([ + expect.stringContaining( + "has no durable checkpoint and later generations are already imported", + ), + ]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([ + { query: "newer generation" }, + ]); + }); + + it("defers host event import when a source contains invalid rows", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + await fs.writeFile( + eventPath, + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "valid before malformed", + resultCount: 0, + results: [], + })}\n${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:01.000Z", + })}\n{malformed\n`, + "utf8", + ); + + const result = await hostEventsMigration().migrateLegacyState(migrationParams()); + + expect(result.changes).toEqual([]); + expect(result.warnings).toEqual([ + expect.stringContaining("Skipped invalid Memory Core host event"), + expect.stringContaining("Skipped malformed Memory Core host event"), + expect.stringContaining("invalid rows still require repair"), + ]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toEqual([]); + await expect(fs.access(eventPath)).resolves.toBeUndefined(); + await expect(fs.access(`${eventPath}.migrated`)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("does not import newer host event generations before an older source is repaired", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const archivedPath = `${eventPath}.migrated`; + const event = (query: string) => ({ + type: "memory.recall.recorded" as const, + timestamp: "2026-07-01T00:00:00.000Z", + query, + resultCount: 0, + results: [], + }); + await fs.writeFile(archivedPath, "{malformed\n", "utf8"); + await fs.writeFile(eventPath, `${JSON.stringify(event("newer generation"))}\n`, "utf8"); + const migration = hostEventsMigration(); + + const blocked = await migration.migrateLegacyState(migrationParams()); + + expect(blocked.changes).toEqual([]); + expect(blocked.warnings).toEqual([ + expect.stringContaining("Skipped malformed Memory Core host event"), + expect.stringContaining("invalid rows still require repair"), + ]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toEqual([]); + await expect(fs.access(eventPath)).resolves.toBeUndefined(); + + await fs.writeFile( + archivedPath, + `${JSON.stringify(event("repaired older generation"))}\n`, + "utf8", + ); + const repaired = await migration.migrateLegacyState(migrationParams()); + + expect(repaired.warnings).toEqual([]); + expect(repaired.changes).toEqual([ + expect.stringContaining("Recovered 1 later Memory Core host event row"), + "Migrated Memory Core host events -> SQLite plugin state (1 new row(s))", + expect.stringContaining("Archived Memory Core host events legacy source"), + ]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([ + { query: "repaired older generation" }, + { query: "newer generation" }, + ]); + }); + + it("imports host events after malformed rows are repaired", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const event = { + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "stable after repair", + resultCount: 0, + results: [], + }; + await fs.writeFile(eventPath, `{malformed\n${JSON.stringify(event)}\n`, "utf8"); + const migration = hostEventsMigration(); + + await migration.migrateLegacyState(migrationParams()); + await fs.writeFile( + eventPath, + ` ${JSON.stringify({ + results: [], + resultCount: 0, + query: event.query, + timestamp: event.timestamp, + type: event.type, + })} \n`, + "utf8", + ); + const repaired = await migration.migrateLegacyState(migrationParams()); + + expect(repaired.warnings).toEqual([]); + expect(repaired.changes).toEqual([ + "Migrated Memory Core host events -> SQLite plugin state (1 new row(s))", + expect.stringContaining("Archived Memory Core host events legacy source"), + ]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([ + { query: "stable after repair" }, + ]); + await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined(); + }); + + it("preserves legacy host event append order when timestamps move backward", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const events = [ + { + type: "memory.recall.recorded", + timestamp: "2026-07-02T00:00:00.000Z", + query: "appended first", + resultCount: 0, + results: [], + }, + { + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "appended last after clock rollback", + resultCount: 0, + results: [], + }, + ]; + await fs.writeFile(eventPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`); + + const result = await hostEventsMigration().migrateLegacyState(migrationParams()); + + expect(result.warnings).toEqual([]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([ + { query: "appended first" }, + { query: "appended last after clock rollback" }, + ]); + await expect( + readMemoryHostEventRecords({ workspaceDir, env, limit: 1 }), + ).resolves.toMatchObject([{ query: "appended last after clock rollback" }]); + }); + + it.runIf(process.platform !== "win32")( + "canonicalizes and deduplicates aliased legacy host event sources", + async () => { + const workspaceAlias = path.join(rootDir, "workspace-alias"); + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + await fs.symlink(workspaceDir, workspaceAlias); + await fs.writeFile( + eventPath, + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "canonical alias", + resultCount: 0, + results: [], + })}\n`, + "utf8", + ); + const params = migrationParams({ + agents: { + list: [ + { id: "main", workspace: workspaceDir }, + { id: "alias", workspace: workspaceAlias }, + ], + }, + }); + const migration = hostEventsMigration(); + + await expect(migration.detectLegacyState(params)).resolves.toEqual({ + preview: [expect.stringContaining("Memory Core host events")], + }); + const result = await migration.migrateLegacyState(params); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated Memory Core host events -> SQLite plugin state (1 new row(s))", + expect.stringContaining("Archived Memory Core host events legacy source"), + ]); + await expect( + readMemoryHostEventRecords({ workspaceDir: workspaceAlias, env }), + ).resolves.toMatchObject([{ query: "canonical alias" }]); + await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined(); + }, + ); + + it.runIf(process.platform !== "win32")( + "rejects legacy host events beneath symlinked workspace parents", + async () => { + const externalMemoryDir = await fs.mkdtemp( + path.join(os.tmpdir(), "openclaw-memory-core-external-events-"), + ); + const externalEventPath = path.join(externalMemoryDir, ".dreams", "events.jsonl"); + try { + await fs.rm(path.join(workspaceDir, "memory"), { recursive: true }); + await fs.mkdir(path.dirname(externalEventPath), { recursive: true }); + await fs.writeFile( + externalEventPath, + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "outside workspace", + resultCount: 0, + results: [], + })}\n`, + "utf8", + ); + await fs.symlink(externalMemoryDir, path.join(workspaceDir, "memory")); + + await expect(hostEventsMigration().detectLegacyState(migrationParams())).resolves.toEqual({ + preview: [expect.stringContaining("requires safe-path repair")], + }); + const result = await hostEventsMigration().migrateLegacyState(migrationParams()); + + expect(result.changes).toEqual([]); + expect(result.warnings).toEqual([ + expect.stringContaining("Skipped unsafe Memory Core host event source"), + ]); + await expect(fs.readFile(externalEventPath, "utf8")).resolves.toContain( + "outside workspace", + ); + await expect(fs.access(`${externalEventPath}.migrated`)).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + await fs.rm(externalMemoryDir, { recursive: true, force: true }); + } + }, + ); + + it("imports the newest retained tail from an oversized legacy host event log", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const events = Array.from({ length: 10_002 }, (_, index) => ({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: `oversized-${index}`, + resultCount: 0, + results: [], + })); + await fs.writeFile(eventPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`); + + const result = await hostEventsMigration().migrateLegacyState(migrationParams()); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated Memory Core host events -> SQLite plugin state (10000 new row(s))", + expect.stringContaining("Archived Memory Core host events legacy source"), + ]); + const imported = await readMemoryHostEventRecords({ workspaceDir, env }); + expect(imported).toHaveLength(10_000); + expect(imported[0]).toMatchObject({ query: "oversized-2" }); + expect(imported.at(-1)).toMatchObject({ query: "oversized-10001" }); + await context() + .openPluginStateKeyedStore({ + namespace: "memory-host.event-migration-checkpoints", + maxEntries: 10_000, + overflowPolicy: "reject-new", + }) + .clear(); + const retriedWithoutCheckpoint = + await hostEventsMigration().migrateLegacyState(migrationParams()); + expect(retriedWithoutCheckpoint.warnings).toEqual([]); + const afterCheckpointRetry = await readMemoryHostEventRecords({ workspaceDir, env }); + expect(afterCheckpointRetry[0]).toMatchObject({ query: "oversized-2" }); + expect(afterCheckpointRetry.at(-1)).toMatchObject({ query: "oversized-10001" }); + await fs.writeFile( + eventPath, + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-07-02T00:00:00.000Z", + query: "newer recreated generation", + resultCount: 0, + results: [], + })}\n`, + ); + const repeated = await hostEventsMigration().migrateLegacyState(migrationParams()); + expect(repeated.warnings).toEqual([]); + expect(repeated.changes[0]).toContain("1 new row"); + const afterRepeated = await readMemoryHostEventRecords({ workspaceDir, env }); + expect(afterRepeated).toHaveLength(10_000); + expect(afterRepeated[0]).toMatchObject({ query: "oversized-3" }); + expect(afterRepeated.at(-1)).toMatchObject({ query: "newer recreated generation" }); + await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined(); + await expect(fs.access(`${eventPath}.migrated.2`)).resolves.toBeUndefined(); + }); + + it("leaves legacy host events in place when plugin-wide SQLite capacity is exhausted", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + await fs.writeFile( + eventPath, + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: "sqlite capacity", + resultCount: 0, + results: [], + })}\n`, + "utf8", + ); + const params = migrationParams(); + params.context = { + ...params.context, + getPluginStateCapacity: () => ({ liveEntries: 50_000, maxEntries: 50_000 }), + }; + + const result = await hostEventsMigration().migrateLegacyState(params); + + expect(result.changes).toEqual([]); + expect(result.warnings).toEqual([expect.stringContaining("no room for its workspace cursor")]); + await expect(fs.access(eventPath)).resolves.toBeUndefined(); + await expect(fs.access(`${eventPath}.migrated`)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("reserves plugin-wide capacity for the migrated workspace cursor and checkpoint", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + const events = Array.from({ length: 3 }, (_, index) => ({ + type: "memory.recall.recorded", + timestamp: "2026-07-01T00:00:00.000Z", + query: `capacity-${index}`, + resultCount: 0, + results: [], + })); + await fs.writeFile(eventPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`); + const params = migrationParams(); + params.context = { + ...params.context, + getPluginStateCapacity: () => ({ liveEntries: 49_997, maxEntries: 50_000 }), + }; + + const result = await hostEventsMigration().migrateLegacyState(params); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated Memory Core host events -> SQLite plugin state (1 new row(s))", + expect.stringContaining("Archived Memory Core host events legacy source"), + ]); + await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([ + { query: "capacity-2" }, + ]); + const cursors = await context() + .openPluginStateKeyedStore<{ kind: "cursor"; lastSequence: number }>({ + namespace: "memory-host.event-cursors", + maxEntries: 1_000, + }) + .entries(); + expect(cursors).toHaveLength(1); + const checkpoints = await context() + .openPluginStateKeyedStore({ + namespace: "memory-host.event-migration-checkpoints", + maxEntries: 10_000, + overflowPolicy: "reject-new", + }) + .entries(); + expect(checkpoints).toHaveLength(1); + }); + + it("retires an empty legacy memory host event source without claiming an import", async () => { + const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl"); + await fs.writeFile(eventPath, "\n", "utf8"); + + const result = await hostEventsMigration().migrateLegacyState(migrationParams()); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Retired empty Memory Core host events legacy source", + expect.stringContaining("Archived Memory Core host events legacy source"), + ]); + const entries = await context() + .openPluginStateKeyedStore({ namespace: "memory-host.events", maxEntries: 10_000 }) + .entries(); + expect(entries).toEqual([]); + await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined(); + }); + it("imports persistent legacy dreaming state and ignores transient locks", async () => { const dreamsDir = path.join(workspaceDir, "memory", ".dreams"); const dailyPath = path.join(dreamsDir, "daily-ingestion.json"); @@ -542,7 +1192,7 @@ describe("memory-core doctor dreaming migration", () => { ); await fs.writeFile(lockPath, `${process.pid}:${Date.now()}\n`, "utf8"); - const migration = requireStateMigration(0); + const migration = dreamingStateMigration(); const preview = await migration.detectLegacyState(migrationParams()); expect(preview?.preview).toEqual([ expect.stringContaining("Memory Core daily ingestion"), @@ -624,7 +1274,7 @@ describe("memory-core doctor dreaming migration", () => { const recallPath = path.join(workspaceDir, "memory", ".dreams", "short-term-recall.json"); await fs.writeFile(recallPath, "{", "utf8"); - const result = await requireStateMigration(0).migrateLegacyState(migrationParams()); + const result = await dreamingStateMigration().migrateLegacyState(migrationParams()); expect(result.changes).toEqual([]); expect(result.warnings).toEqual([ @@ -666,10 +1316,10 @@ describe("memory-core doctor dreaming migration", () => { ); const config = { agents: { list: [{ id: "main", default: true }] } }; - const preview = await requireStateMigration(0).detectLegacyState(migrationParams(config)); + const preview = await dreamingStateMigration().detectLegacyState(migrationParams(config)); expect(preview?.preview).toEqual([expect.stringContaining("Memory Core short-term recall")]); - const result = await requireStateMigration(0).migrateLegacyState(migrationParams(config)); + const result = await dreamingStateMigration().migrateLegacyState(migrationParams(config)); expect(result.warnings).toEqual([]); expect(result.changes).toEqual([ diff --git a/extensions/memory-core/doctor-contract-api.ts b/extensions/memory-core/doctor-contract-api.ts index 0bbdcfd5540..c698758ba7d 100644 --- a/extensions/memory-core/doctor-contract-api.ts +++ b/extensions/memory-core/doctor-contract-api.ts @@ -6,7 +6,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { reclaimDefinitelyStaleFileLock } from "openclaw/plugin-sdk/file-lock"; -import { resolveUserPath } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; +import { resolveUserPath, root } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; import { ensureMemoryIndexSchema, loadSqliteVecExtension, @@ -19,10 +19,15 @@ import { requireNodeSqlite, } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import { resolveMemoryDreamingWorkspaces } from "openclaw/plugin-sdk/memory-core-host-status"; +import { + normalizeMemoryHostEventRecordForStorage, + resolveMemoryHostEventLogPath, +} from "openclaw/plugin-sdk/memory-host-events"; import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; import { archiveLegacyStateSource, legacyStateFileExists, + type PluginDoctorStateMigrationContext, type PluginDoctorStateMigration, } from "openclaw/plugin-sdk/runtime-doctor"; import { @@ -67,6 +72,70 @@ type LegacySource = { filePath: string; }; +type LegacyMemoryHostEventSource = + | { + kind: "ready"; + workspaceDir: string; + filePath: string; + relativePath: string; + root: Awaited>; + storage: "active" | "claim" | "archive"; + archiveRelativePath?: string; + generationKey?: string; + } + | { + kind: "rejected"; + workspaceDir: string; + filePath: string; + reason: string; + }; + +type ReadyLegacyMemoryHostEventSource = Extract; + +type StoredMemoryHostEvent = { + kind: "event"; + event: NonNullable>; + recordedAt: number; + sequence: number; +}; + +type StoredMemoryHostCursor = { + kind: "cursor"; + lastSequence: number; +}; + +type StoredMemoryHostMigrationCheckpoint = { + kind: "migration-checkpoint"; + contentHash: string; + recordCount: number; + sequenceBase: number; + size: number; +}; + +const MEMORY_HOST_EVENTS_NAMESPACE = "memory-host.events"; +const MEMORY_HOST_EVENT_CURSORS_NAMESPACE = "memory-host.event-cursors"; +const MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS_NAMESPACE = "memory-host.event-migration-checkpoints"; +// Keep migration aligned with event-store.ts retention so legacy import cannot +// consume memory-core's plugin-wide budget or starve sibling state namespaces. +const MAX_MEMORY_HOST_EVENTS = 10_000; +const MAX_MEMORY_HOST_EVENT_CURSORS = 1_000; +const MAX_MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS = 10_000; +const MAX_LEGACY_MEMORY_HOST_EVENT_VALUE_BYTES = 65_536; +const LEGACY_MEMORY_HOST_SEQUENCE_BASE = Number.MIN_SAFE_INTEGER; + +function normalizeMemoryHostWorkspaceKey(workspaceDir: string): string { + const resolved = path.resolve(workspaceDir).replace(/\\/g, "/"); + return process.platform === "win32" ? resolved.toLowerCase() : resolved; +} + +function memoryHostWorkspacePrefix(workspaceDir: string): string { + return crypto + .createHash("sha256") + .update(normalizeMemoryHostWorkspaceKey(workspaceDir)) + .digest("hex") + .slice(0, 24); +} + type LegacyMemorySidecarSource = { agentId: string; legacyPath: string; @@ -1175,7 +1244,709 @@ async function migrateSource(source: LegacySource): Promise { return await migratePhaseSignals(source); } +async function collectLegacyMemoryHostEventSources( + config: unknown, + env: NodeJS.ProcessEnv, +): Promise { + const sources: LegacyMemoryHostEventSource[] = []; + const seenWorkspaces = new Set(); + for (const workspaceDir of resolveConfiguredWorkspaces(config, env)) { + let canonicalWorkspaceDir = path.resolve(workspaceDir); + let filePath = resolveMemoryHostEventLogPath(canonicalWorkspaceDir); + try { + const workspaceRoot = await root(workspaceDir, { + hardlinks: "reject", + // Legacy doctor import previously read the complete JSONL source. + maxBytes: Number.MAX_SAFE_INTEGER, + mkdir: false, + symlinks: "reject", + }); + canonicalWorkspaceDir = workspaceRoot.rootReal; + if (seenWorkspaces.has(canonicalWorkspaceDir)) { + continue; + } + seenWorkspaces.add(canonicalWorkspaceDir); + filePath = resolveMemoryHostEventLogPath(canonicalWorkspaceDir); + const relativePath = path.relative(canonicalWorkspaceDir, filePath); + const directoryRelativePath = path.dirname(relativePath); + if (!(await workspaceRoot.exists(directoryRelativePath))) { + continue; + } + const directoryStat = await workspaceRoot.stat(directoryRelativePath); + if (!directoryStat.isDirectory) { + continue; + } + const baseName = path.basename(relativePath).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const archivePattern = new RegExp(`^${baseName}\\.migrated(?:\\.([2-9]|[1-9][0-9]+))?$`, "u"); + const claimPattern = new RegExp( + `^\\.${baseName}\\.doctor-importing(?:\\.([2-9]|[1-9][0-9]+))?$`, + "u", + ); + const entries = await fs.readdir(path.join(workspaceRoot.rootReal, directoryRelativePath)); + const candidates: Array<{ + entry: string; + storage: "active" | "claim" | "archive"; + generation: bigint | undefined; + }> = []; + for (const entry of entries) { + if (entry === path.basename(relativePath)) { + candidates.push({ entry, storage: "active", generation: undefined }); + continue; + } + const claim = claimPattern.exec(entry); + if (claim) { + candidates.push({ entry, storage: "claim", generation: BigInt(claim[1] ?? "1") }); + continue; + } + const archive = archivePattern.exec(entry); + if (archive) { + candidates.push({ entry, storage: "archive", generation: BigInt(archive[1] ?? "1") }); + } + } + candidates.sort((left, right) => { + if (left.generation === undefined) { + return 1; + } + if (right.generation === undefined) { + return -1; + } + return left.generation < right.generation ? -1 : left.generation > right.generation ? 1 : 0; + }); + for (const candidate of candidates) { + const candidateRelativePath = path.join(directoryRelativePath, candidate.entry); + const stat = await workspaceRoot.stat(candidateRelativePath); + if (!stat.isFile) { + continue; + } + const generationText = candidate.generation?.toString(); + const generationKey = generationText?.padStart(20, "0"); + sources.push({ + kind: "ready", + workspaceDir: canonicalWorkspaceDir, + filePath: path.join(canonicalWorkspaceDir, candidateRelativePath), + relativePath: candidateRelativePath, + root: workspaceRoot, + storage: candidate.storage, + ...(candidate.storage === "active" + ? {} + : { + archiveRelativePath: `${relativePath}.migrated${candidate.generation === 1n ? "" : `.${candidate.generation}`}`, + generationKey, + }), + }); + } + } catch (error) { + const code = (error as { code?: unknown }).code; + if (code === "ENOENT" || code === "ENOTDIR" || code === "not-found") { + continue; + } + if (!seenWorkspaces.has(canonicalWorkspaceDir)) { + seenWorkspaces.add(canonicalWorkspaceDir); + } + sources.push({ + kind: "rejected", + workspaceDir: canonicalWorkspaceDir, + filePath, + reason: String(error), + }); + } + } + return sources; +} + +async function resolveMemoryHostEventArchivePath( + source: ReadyLegacyMemoryHostEventSource, +): Promise<{ archiveRelativePath: string; claimRelativePath: string; generationKey: string }> { + const activeRelativePath = path.relative( + source.workspaceDir, + resolveMemoryHostEventLogPath(source.workspaceDir), + ); + const directoryPath = path.join(source.root.rootReal, path.dirname(activeRelativePath)); + const baseName = path.basename(activeRelativePath).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const archivePattern = new RegExp(`^${baseName}\\.migrated(?:\\.([2-9]|[1-9][0-9]+))?$`, "u"); + const claimPattern = new RegExp( + `^\\.${baseName}\\.doctor-importing(?:\\.([2-9]|[1-9][0-9]+))?$`, + "u", + ); + let latestGeneration = 0n; + for (const entry of await fs.readdir(directoryPath)) { + const match = archivePattern.exec(entry) ?? claimPattern.exec(entry); + if (!match) { + continue; + } + const generation = BigInt(match[1] ?? "1"); + if (generation > latestGeneration) { + latestGeneration = generation; + } + } + const generation = latestGeneration + 1n; + const generationText = generation.toString(); + if (generationText.length > 20) { + throw new RangeError("Memory Core host event archive generation is too large"); + } + const generationSuffix = generation === 1n ? "" : `.${generation}`; + return { + archiveRelativePath: `${activeRelativePath}.migrated${generationSuffix}`, + claimRelativePath: path.join( + path.dirname(activeRelativePath), + `.${path.basename(activeRelativePath)}.doctor-importing${generationSuffix}`, + ), + // Fixed-width decimal order keeps key-range reads chronological across archives. + generationKey: generationText.padStart(20, "0"), + }; +} + +function memoryHostMigrationCheckpointKey(source: ReadyLegacyMemoryHostEventSource): string { + if (!source.generationKey) { + throw new Error(`Missing Memory Core host event archive generation for ${source.filePath}`); + } + return `${memoryHostWorkspacePrefix(source.workspaceDir)}:archive:${source.generationKey}`; +} + +function memoryHostMigrationSnapshot(raw: string, recordCount: number, sequenceBase: number) { + return { + kind: "migration-checkpoint" as const, + contentHash: crypto.createHash("sha256").update(raw).digest("hex"), + recordCount, + sequenceBase, + size: Buffer.byteLength(raw, "utf8"), + }; +} + +function isMemoryHostMigrationCheckpoint( + value: StoredMemoryHostMigrationCheckpoint | undefined, +): value is StoredMemoryHostMigrationCheckpoint { + return ( + value?.kind === "migration-checkpoint" && + typeof value.contentHash === "string" && + Number.isSafeInteger(value.recordCount) && + value.recordCount >= 0 && + Number.isSafeInteger(value.sequenceBase) && + Number.isSafeInteger(value.size) && + value.size >= 0 + ); +} + +async function memoryHostEventSourceNeedsMigration(params: { + source: ReadyLegacyMemoryHostEventSource; + context: PluginDoctorStateMigrationContext; +}): Promise { + if (params.source.storage !== "archive") { + return true; + } + const checkpoint = await params.context + .openPluginStateKeyedStore({ + namespace: MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS_NAMESPACE, + maxEntries: MAX_MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS, + overflowPolicy: "reject-new", + }) + .lookup(memoryHostMigrationCheckpointKey(params.source)); + if (!isMemoryHostMigrationCheckpoint(checkpoint)) { + return true; + } + const raw = await params.source.root.readText(params.source.relativePath); + return ( + Buffer.byteLength(raw, "utf8") !== checkpoint.size || + crypto.createHash("sha256").update(raw).digest("hex") !== checkpoint.contentHash + ); +} + +async function finalizeLegacyMemoryHostEventSource(params: { + source: ReadyLegacyMemoryHostEventSource; + changes: string[]; + warnings: string[]; +}): Promise { + if (params.source.storage === "archive") { + return true; + } + const archivedRelativePath = params.source.archiveRelativePath; + if (!archivedRelativePath) { + throw new Error(`Missing Memory Core host event archive path for ${params.source.filePath}`); + } + try { + await params.source.root.move(params.source.relativePath, archivedRelativePath); + params.changes.push( + `Archived Memory Core host events legacy source -> ${path.join(params.source.workspaceDir, archivedRelativePath)}`, + ); + return true; + } catch (error) { + params.warnings.push( + `Failed archiving Memory Core host events legacy source: ${String(error)}`, + ); + return false; + } +} + +async function restoreClaimedMemoryHostEventSource(params: { + source: ReadyLegacyMemoryHostEventSource; + activeRelativePath: string; + warnings: string[]; +}): Promise { + try { + if (!(await params.source.root.exists(params.source.relativePath))) { + return; + } + if (!(await params.source.root.exists(params.activeRelativePath))) { + await params.source.root.move(params.source.relativePath, params.activeRelativePath); + return; + } + params.warnings.push( + `Left claimed Memory Core host events at ${params.source.filePath} because an old writer recreated the active source`, + ); + } catch (error) { + params.warnings.push( + `Failed restoring claimed Memory Core host events ${params.source.filePath}: ${String(error)}`, + ); + } +} + +async function migrateLegacyMemoryHostEventSource(params: { + source: ReadyLegacyMemoryHostEventSource; + context: PluginDoctorStateMigrationContext; + changes: string[]; + warnings: string[]; +}): Promise<"completed" | "blocked"> { + const activeRelativePath = path.relative( + params.source.workspaceDir, + resolveMemoryHostEventLogPath(params.source.workspaceDir), + ); + let source = params.source; + let restoreNewClaim = false; + let claimFinalized = source.storage === "archive"; + if (source.storage === "active") { + const generation = await resolveMemoryHostEventArchivePath(source); + await source.root.move(source.relativePath, generation.claimRelativePath); + source = { + ...source, + filePath: path.join(source.workspaceDir, generation.claimRelativePath), + relativePath: generation.claimRelativePath, + storage: "claim", + archiveRelativePath: generation.archiveRelativePath, + generationKey: generation.generationKey, + }; + restoreNewClaim = true; + } + + try { + if (!source.generationKey) { + throw new Error(`Missing Memory Core host event generation for ${source.filePath}`); + } + const warningStart = params.warnings.length; + const raw = await source.root.readText(source.relativePath); + const prefix = memoryHostWorkspacePrefix(source.workspaceDir); + const records: Array<{ + digest: string; + ordinal: number; + value: StoredMemoryHostEvent; + }> = []; + for (const [lineIndex, line] of raw.split(/\r?\n/u).entries()) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + let event: unknown; + try { + event = JSON.parse(trimmed) as unknown; + } catch (error) { + params.warnings.push( + `Skipped malformed Memory Core host event at ${source.filePath}:${lineIndex + 1}: ${String(error)}`, + ); + continue; + } + const normalizedEvent = normalizeMemoryHostEventRecordForStorage(event); + if (!normalizedEvent) { + params.warnings.push( + `Skipped invalid Memory Core host event at ${source.filePath}:${lineIndex + 1}`, + ); + continue; + } + const canonicalEvent = JSON.stringify(normalizedEvent); + const parsedTimestamp = Date.parse(normalizedEvent.timestamp); + const recordedAt = Number.isSafeInteger(parsedTimestamp) ? parsedTimestamp : 0; + const ordinal = records.length; + const digest = crypto.createHash("sha256").update(canonicalEvent).digest("hex"); + const value: StoredMemoryHostEvent = { + kind: "event", + event: normalizedEvent, + recordedAt, + sequence: LEGACY_MEMORY_HOST_SEQUENCE_BASE + ordinal + 1, + }; + if ( + Buffer.byteLength(JSON.stringify(value), "utf8") > MAX_LEGACY_MEMORY_HOST_EVENT_VALUE_BYTES + ) { + params.warnings.push( + `Skipped oversized Memory Core host event at ${source.filePath}:${lineIndex + 1}`, + ); + continue; + } + records.push({ + // Runtime keys use the adjacent `1:` range. Generation plus valid-row + // ordinal makes interrupted imports and later raw-archive recovery stable. + digest, + ordinal, + value, + }); + } + if (params.warnings.length > warningStart) { + params.warnings.push( + "Left Memory Core host events legacy source in place because invalid rows still require repair", + ); + return "blocked"; + } + + const checkpointStore = + params.context.openPluginStateKeyedStore({ + namespace: MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS_NAMESPACE, + maxEntries: MAX_MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS, + overflowPolicy: "reject-new", + }); + // Plugin-wide shedding is namespace-local. `reject-new` therefore keeps + // retained raw-archive checkpoints out of every sibling namespace's + // eviction budget and fails before this namespace can rotate its own rows. + const checkpointKey = memoryHostMigrationCheckpointKey(source); + const checkpointValue = await checkpointStore.lookup(checkpointKey); + const previousCheckpoint = isMemoryHostMigrationCheckpoint(checkpointValue) + ? checkpointValue + : undefined; + if (source.storage === "archive" && previousCheckpoint) { + const bytes = Buffer.from(raw, "utf8"); + const prefixHash = crypto + .createHash("sha256") + .update(bytes.subarray(0, previousCheckpoint.size)) + .digest("hex"); + if ( + bytes.length < previousCheckpoint.size || + prefixHash !== previousCheckpoint.contentHash || + records.length < previousCheckpoint.recordCount + ) { + params.warnings.push( + `Skipped Memory Core host event recovery because ${source.filePath} changed other than by append; left the archive in place`, + ); + return "blocked"; + } + } + const firstCandidateOrdinal = + source.storage === "archive" && previousCheckpoint ? previousCheckpoint.recordCount : 0; + const candidateRecords = records.slice(firstCandidateOrdinal); + const store = params.context.openPluginStateKeyedStore({ + namespace: MEMORY_HOST_EVENTS_NAMESPACE, + maxEntries: MAX_MEMORY_HOST_EVENTS, + }); + const existingEntries = await store.entries(); + const existingKeys = new Set(existingEntries.map((entry) => entry.key)); + const latestLegacySequence = existingEntries.reduce( + (latest, entry) => + entry.value.sequence < 0 ? Math.max(latest, entry.value.sequence) : latest, + LEGACY_MEMORY_HOST_SEQUENCE_BASE, + ); + const legacyKeyPrefix = `${prefix}:event:0:s:`; + const existingByIdentity = new Map( + existingEntries.flatMap((entry) => { + if (!entry.key.startsWith(legacyKeyPrefix) || entry.value.sequence >= 0) { + return []; + } + const parts = entry.key.split(":"); + return parts.length === 8 ? [[`${parts[5]}:${parts[6]}:${parts[7]}`, entry] as const] : []; + }), + ); + const existingSourceBase = candidateRecords.flatMap((record) => { + const identity = `${source.generationKey}:${record.ordinal.toString().padStart(16, "0")}:${record.digest}`; + const existing = existingByIdentity.get(identity); + return existing ? [existing.value.sequence - (record.ordinal + 1)] : []; + })[0]; + const laterGenerationExists = existingEntries.some((entry) => { + if (!entry.key.startsWith(legacyKeyPrefix) || entry.value.sequence >= 0) { + return false; + } + const generationKey = entry.key.split(":")[5]; + return generationKey !== undefined && generationKey > source.generationKey!; + }); + if ( + source.storage === "archive" && + !previousCheckpoint && + existingSourceBase === undefined && + laterGenerationExists + ) { + params.warnings.push( + `Skipped Memory Core host event recovery because ${source.filePath} has no durable checkpoint and later generations are already imported; left the archive in place`, + ); + return "blocked"; + } + const sourceSequenceBase = + previousCheckpoint?.sequenceBase ?? existingSourceBase ?? latestLegacySequence; + let nextSequence = latestLegacySequence; + const sequencedRecords = candidateRecords.map((record) => { + const ordinalKey = record.ordinal.toString().padStart(16, "0"); + const identity = `${source.generationKey}:${ordinalKey}:${record.digest}`; + const existing = existingByIdentity.get(identity); + if (existing) { + return { + ...record, + key: existing.key, + value: { ...record.value, sequence: existing.value.sequence }, + }; + } + const sequence = previousCheckpoint + ? (nextSequence += 1) + : sourceSequenceBase + record.ordinal + 1; + const sequenceKey = (sequence - LEGACY_MEMORY_HOST_SEQUENCE_BASE) + .toString() + .padStart(16, "0"); + return { + ...record, + key: `${legacyKeyPrefix}${sequenceKey}:${identity}`, + value: { ...record.value, sequence }, + }; + }); + const sourceKeys = new Set(sequencedRecords.map((record) => record.key)); + if ( + sequencedRecords.some( + (record) => !Number.isSafeInteger(record.value.sequence) || record.value.sequence >= 0, + ) + ) { + params.warnings.push( + "Skipped Memory Core host event migration because legacy sequence capacity is exhausted; left legacy source in place", + ); + return "blocked"; + } + const nativeCount = existingEntries.filter((entry) => entry.value.sequence >= 0).length; + const legacyRetentionLimit = Math.max(0, MAX_MEMORY_HOST_EVENTS - nativeCount); + const combinedLegacy = [ + ...existingEntries + .filter((entry) => entry.value.sequence < 0 && !sourceKeys.has(entry.key)) + .map((entry) => ({ key: entry.key, sequence: entry.value.sequence })), + ...sequencedRecords.map((record) => ({ + key: record.key, + sequence: record.value.sequence, + })), + ].toSorted( + (left, right) => left.sequence - right.sequence || left.key.localeCompare(right.key), + ); + const desiredLegacyKeys = new Set( + legacyRetentionLimit === 0 + ? [] + : combinedLegacy.slice(-legacyRetentionLimit).map((record) => record.key), + ); + let retainedRecords = sequencedRecords.filter((record) => desiredLegacyKeys.has(record.key)); + const capacity = params.context.getPluginStateCapacity?.(); + if (!capacity) { + params.warnings.push( + "Skipped Memory Core host event migration because plugin-wide SQLite capacity is unavailable; left legacy source in place", + ); + return "blocked"; + } + const pluginRemainingCapacity = Math.max(0, capacity.maxEntries - capacity.liveEntries); + const cursorStore = params.context.openPluginStateKeyedStore({ + namespace: MEMORY_HOST_EVENT_CURSORS_NAMESPACE, + maxEntries: MAX_MEMORY_HOST_EVENT_CURSORS, + }); + const cursorKey = `${prefix}:cursor`; + const existingCursor = await cursorStore.lookup(cursorKey); + const cursorCapacity = candidateRecords.length > 0 && existingCursor?.kind !== "cursor" ? 1 : 0; + if (cursorCapacity > pluginRemainingCapacity) { + params.warnings.push( + "Skipped Memory Core host event migration because SQLite plugin state has no room for its workspace cursor; left legacy source in place", + ); + return "blocked"; + } + const checkpointCapacity = checkpointValue ? 0 : 1; + if ( + checkpointCapacity > 0 && + (await checkpointStore.entries()).length >= MAX_MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS + ) { + // Checkpoints use reject-new and never expire while their raw archives remain. + // Stop before import/archive once durable processed-generation capacity is full. + params.warnings.push( + "Skipped Memory Core host event migration because durable raw-archive checkpoint capacity is exhausted; left legacy source in place", + ); + return "blocked"; + } + if (cursorCapacity + checkpointCapacity > pluginRemainingCapacity) { + params.warnings.push( + "Skipped Memory Core host event migration because SQLite plugin state has no room for its raw-archive checkpoint; left legacy source in place", + ); + return "blocked"; + } + const importEntries = params.context.importPluginStateEntries; + if (candidateRecords.length > 0 && !importEntries) { + params.warnings.push( + "Skipped Memory Core host event migration because retention-aware SQLite import is unavailable; left legacy source in place", + ); + return "blocked"; + } + let retainedKeys = new Set(retainedRecords.map((record) => record.key)); + let missing = retainedRecords.filter((record) => !existingKeys.has(record.key)); + let replaceableLegacyRows = existingEntries.filter( + (entry) => entry.value.sequence < 0 && !retainedKeys.has(entry.key), + ).length; + const reservedCapacity = cursorCapacity + checkpointCapacity; + const eventCapacity = pluginRemainingCapacity - reservedCapacity + replaceableLegacyRows; + const retentionDeficit = Math.max(0, missing.length - eventCapacity); + if (retentionDeficit > 0) { + retainedRecords = retainedRecords.slice(Math.min(retentionDeficit, retainedRecords.length)); + retainedKeys = new Set(retainedRecords.map((record) => record.key)); + missing = retainedRecords.filter((record) => !existingKeys.has(record.key)); + replaceableLegacyRows = existingEntries.filter( + (entry) => entry.value.sequence < 0 && !retainedKeys.has(entry.key), + ).length; + } + const availableEventCapacity = + pluginRemainingCapacity - reservedCapacity + replaceableLegacyRows; + if (missing.length > availableEventCapacity) { + params.warnings.push( + `Skipped Memory Core host event migration because SQLite plugin state has room for ${availableEventCapacity} of ${missing.length} missing rows after reserving its cursor and raw-archive checkpoint; left legacy source in place`, + ); + return "blocked"; + } + if (cursorCapacity > 0) { + const lastSequence = existingEntries.reduce( + (maximum, entry) => + Number.isSafeInteger(entry.value.sequence) + ? Math.max(maximum, entry.value.sequence) + : maximum, + 0, + ); + await cursorStore.register(cursorKey, { kind: "cursor", lastSequence }); + const registeredCursor = await cursorStore.lookup(cursorKey); + if (registeredCursor?.kind !== "cursor") { + params.warnings.push( + "Skipped Memory Core host event migration because its workspace cursor could not be verified; left legacy source in place", + ); + return "blocked"; + } + } + importEntries?.( + { namespace: MEMORY_HOST_EVENTS_NAMESPACE, maxEntries: MAX_MEMORY_HOST_EVENTS }, + missing.map((record) => ({ + key: record.key, + value: record.value, + // Negative legacy sequence keeps imported rows older than live runtime rows. + createdAt: record.value.sequence, + })), + ); + const importedKeys = new Set((await store.entries()).map((entry) => entry.key)); + const missingKey = retainedRecords.find((record) => !importedKeys.has(record.key))?.key; + if (missingKey) { + params.warnings.push( + `Skipped archiving Memory Core host events because SQLite verification missed ${missingKey}`, + ); + return "blocked"; + } + + if (source.storage === "archive") { + if (missing.length > 0) { + params.changes.push( + `Recovered ${missing.length} later Memory Core host event row(s) from ${source.filePath}`, + ); + } + } else { + params.changes.push( + records.length === 0 + ? "Retired empty Memory Core host events legacy source" + : `Migrated Memory Core host events -> SQLite plugin state (${missing.length} new row(s))`, + ); + claimFinalized = await finalizeLegacyMemoryHostEventSource({ + source, + changes: params.changes, + warnings: params.warnings, + }); + if (!claimFinalized) { + params.changes.pop(); + return "blocked"; + } + } + const checkpoint = memoryHostMigrationSnapshot(raw, records.length, sourceSequenceBase); + await checkpointStore.register(checkpointKey, checkpoint); + const registeredCheckpoint = await checkpointStore.lookup(checkpointKey); + if ( + !isMemoryHostMigrationCheckpoint(registeredCheckpoint) || + registeredCheckpoint.contentHash !== checkpoint.contentHash || + registeredCheckpoint.recordCount !== checkpoint.recordCount || + registeredCheckpoint.sequenceBase !== checkpoint.sequenceBase || + registeredCheckpoint.size !== checkpoint.size + ) { + params.warnings.push( + `Failed verifying Memory Core host event raw-archive checkpoint for ${source.filePath}`, + ); + return "blocked"; + } + if (source.storage !== "archive" && (await source.root.exists(activeRelativePath))) { + params.warnings.push( + "An old writer recreated the Memory Core host event source; rerun openclaw doctor --fix to import the retained rows", + ); + } + return "completed"; + } finally { + if (restoreNewClaim && !claimFinalized) { + await restoreClaimedMemoryHostEventSource({ + source, + activeRelativePath, + warnings: params.warnings, + }); + } + } +} + export const stateMigrations: PluginDoctorStateMigration[] = [ + { + id: "memory-core-host-events-jsonl-to-sqlite", + label: "Memory Core host events", + doctorOnly: true, + async detectLegacyState(params) { + const sources = await collectLegacyMemoryHostEventSources(params.config, params.env); + const pending: LegacyMemoryHostEventSource[] = []; + for (const source of sources) { + if ( + source.kind === "rejected" || + (await memoryHostEventSourceNeedsMigration({ source, context: params.context })) + ) { + pending.push(source); + } + } + if (pending.length === 0) { + return null; + } + return { + preview: pending.map((source) => + source.kind === "ready" + ? `- Memory Core host events: ${source.filePath} -> SQLite plugin state (${MEMORY_HOST_EVENTS_NAMESPACE})` + : `- Memory Core host events: ${source.filePath} requires safe-path repair (${source.reason})`, + ), + }; + }, + async migrateLegacyState(params) { + const changes: string[] = []; + const warnings: string[] = []; + const blockedWorkspaces = new Set(); + for (const source of await collectLegacyMemoryHostEventSources(params.config, params.env)) { + if (blockedWorkspaces.has(source.workspaceDir)) { + continue; + } + if (source.kind === "rejected") { + warnings.push( + `Skipped unsafe Memory Core host event source for ${source.workspaceDir}: ${source.reason}`, + ); + blockedWorkspaces.add(source.workspaceDir); + continue; + } + if (!(await memoryHostEventSourceNeedsMigration({ source, context: params.context }))) { + continue; + } + const result = await migrateLegacyMemoryHostEventSource({ + source, + context: params.context, + changes, + warnings, + }); + if (result === "blocked") { + // Archive generations encode append order. A later generation cannot + // overtake an older source that still needs repair or durable import. + blockedWorkspaces.add(source.workspaceDir); + } + } + return { changes, warnings }; + }, + }, { id: "memory-core-dreams-json-to-sqlite", label: "Memory Core dreaming state", diff --git a/extensions/memory-core/src/memory-events.test.ts b/extensions/memory-core/src/memory-events.test.ts index 44114f51ed8..e46ce61c23a 100644 --- a/extensions/memory-core/src/memory-events.test.ts +++ b/extensions/memory-core/src/memory-events.test.ts @@ -2,9 +2,9 @@ import fs from "node:fs/promises"; import path from "node:path"; import { + appendMemoryHostEvent, readMemoryHostEventRecords, readMemoryHostEvents, - resolveMemoryHostEventLogPath, } from "openclaw/plugin-sdk/memory-host-events"; import { describe, expect, it } from "vitest"; import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js"; @@ -193,20 +193,14 @@ describe("memory host event journal integration", () => { it("keeps legacy dreaming completion events without outcome readable", async () => { const workspaceDir = await createTempWorkspace("memory-core-legacy-dream-events-"); - const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir); - await fs.mkdir(path.dirname(eventLogPath), { recursive: true }); - await fs.writeFile( - eventLogPath, - `${JSON.stringify({ - type: "memory.dream.completed", - timestamp: "2026-04-05T13:00:00.000Z", - phase: "deep", - inlinePath: path.join(workspaceDir, "DREAMS.md"), - lineCount: 2, - storageMode: "inline", - })}\n`, - "utf8", - ); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.dream.completed", + timestamp: "2026-04-05T13:00:00.000Z", + phase: "deep", + inlinePath: path.join(workspaceDir, "DREAMS.md"), + lineCount: 2, + storageMode: "inline", + }); const events = await readMemoryHostEvents({ workspaceDir }); diff --git a/extensions/memory-core/src/public-artifacts.test.ts b/extensions/memory-core/src/public-artifacts.test.ts index 01715810c24..d5813536ad8 100644 --- a/extensions/memory-core/src/public-artifacts.test.ts +++ b/extensions/memory-core/src/public-artifacts.test.ts @@ -2,11 +2,9 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { - appendMemoryHostEvent, - resolveMemoryHostEventLogPath, -} from "openclaw/plugin-sdk/memory-host-events"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events"; +import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../api.js"; import { listMemoryCorePublicArtifacts } from "./public-artifacts.js"; @@ -24,8 +22,15 @@ describe("listMemoryCorePublicArtifacts", () => { await fs.rm(fixtureRoot, { recursive: true, force: true }); }); + afterEach(() => { + resetPluginStateStoreForTests(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + it("lists public workspace artifacts with stable kinds", async () => { const workspaceDir = path.join(fixtureRoot, "workspace-stable-kinds"); + vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot); await fs.mkdir(path.join(workspaceDir, "memory", "dreaming"), { recursive: true }); await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Durable Memory\n", "utf8"); await fs.writeFile( @@ -38,6 +43,8 @@ describe("listMemoryCorePublicArtifacts", () => { "# Dream Report\n", "utf8", ); + const eventStoredAt = Date.parse("2026-04-07T09:30:00.000Z"); + vi.spyOn(Date, "now").mockReturnValue(eventStoredAt); await appendMemoryHostEvent(workspaceDir, { type: "memory.recall.recorded", timestamp: "2026-04-06T12:00:00.000Z", @@ -52,7 +59,12 @@ describe("listMemoryCorePublicArtifacts", () => { }, }; - await expect(listMemoryCorePublicArtifacts({ cfg })).resolves.toEqual([ + const artifacts = await listMemoryCorePublicArtifacts({ cfg }); + const eventArtifact = artifacts.find((artifact) => artifact.kind === "event-log"); + if (!eventArtifact) { + throw new Error("expected memory event export"); + } + expect(artifacts.filter((artifact) => artifact.kind !== "event-log")).toEqual([ { kind: "memory-root", workspaceDir, @@ -77,15 +89,19 @@ describe("listMemoryCorePublicArtifacts", () => { agentIds: ["main"], contentType: "markdown", }, - { - kind: "event-log", - workspaceDir, - relativePath: "memory/.dreams/events.jsonl", - absolutePath: resolveMemoryHostEventLogPath(workspaceDir), - agentIds: ["main"], - contentType: "json", - }, ]); + expect(eventArtifact.relativePath).toMatch( + /^memory\/events\/[a-f0-9]{32}\/memory-host-events\.jsonl$/u, + ); + await expect(fs.readFile(eventArtifact.absolutePath, "utf8")).resolves.toBe( + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-04-06T12:00:00.000Z", + query: "alpha", + resultCount: 0, + results: [], + })}\n`, + ); }); it("ignores lowercase memory root when only the legacy filename exists", async () => { diff --git a/extensions/memory-wiki/src/bridge.test.ts b/extensions/memory-wiki/src/bridge.test.ts index abc1993076f..7dbfa81a2f4 100644 --- a/extensions/memory-wiki/src/bridge.test.ts +++ b/extensions/memory-wiki/src/bridge.test.ts @@ -7,10 +7,6 @@ import { type MemoryPluginPublicArtifact, registerMemoryCapability, } from "openclaw/plugin-sdk/memory-host-core"; -import { - appendMemoryHostEvent, - resolveMemoryHostEventLogPath, -} from "openclaw/plugin-sdk/memory-host-events"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../api.js"; import { syncMemoryWikiBridgeSources } from "./bridge.js"; @@ -436,7 +432,7 @@ describe("syncMemoryWikiBridgeSources", () => { }, }); - await appendMemoryHostEvent(workspaceDir, { + const eventContent = `${JSON.stringify({ type: "memory.recall.recorded", timestamp: "2026-04-05T12:00:00.000Z", query: "bridge events", @@ -449,13 +445,16 @@ describe("syncMemoryWikiBridgeSources", () => { score: 0.8, }, ], - }); + })}\n`; + const eventPath = path.join(workspaceDir, "memory", "events", "memory-host-events.jsonl"); + await fs.mkdir(path.dirname(eventPath), { recursive: true }); + await fs.writeFile(eventPath, eventContent, "utf8"); registerBridgeArtifacts([ { kind: "event-log", workspaceDir, - relativePath: "memory/.dreams/events.jsonl", - absolutePath: resolveMemoryHostEventLogPath(workspaceDir), + relativePath: "memory/events/memory-host-events.jsonl", + absolutePath: eventPath, agentIds: ["main"], contentType: "json", }, diff --git a/extensions/memory-wiki/src/source-page-shared.ts b/extensions/memory-wiki/src/source-page-shared.ts index cc09dc84b93..52d537d13ad 100644 --- a/extensions/memory-wiki/src/source-page-shared.ts +++ b/extensions/memory-wiki/src/source-page-shared.ts @@ -36,6 +36,7 @@ export async function writeImportedSourcePage(params: { vaultRoot: string; syncKey: string; sourcePath: string; + sourceContent?: string; sourceUpdatedAtMs: number; sourceSize: number; renderFingerprint: string; @@ -70,7 +71,7 @@ export async function writeImportedSourcePage(params: { }); const created = !pageStat; const updatedAt = timestampMsToIsoString(params.sourceUpdatedAtMs) ?? new Date().toISOString(); - const raw = await fs.readFile(params.sourcePath, "utf8"); + const raw = params.sourceContent ?? (await fs.readFile(params.sourcePath, "utf8")); const rendered = params.buildRendered(raw, updatedAt); const existing = pageStat ? await readExistingImportedSourcePage(vault, params.pagePath) : ""; const nextRendered = existing ? preserveHumanNotesBlock(rendered, existing) : rendered; diff --git a/package.json b/package.json index c2560672fe7..28a708b7aab 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "openclaw": { "schemaVersions": { "state": 4, - "agent": 11 + "agent": 12 } }, "description": "Multi-channel AI gateway with extensible messaging integrations", diff --git a/scripts/check-database-first-legacy-stores.mjs b/scripts/check-database-first-legacy-stores.mjs index b485fcf39a0..173527c69fd 100644 --- a/scripts/check-database-first-legacy-stores.mjs +++ b/scripts/check-database-first-legacy-stores.mjs @@ -126,7 +126,7 @@ const legacyStorePatterns = [ /\b(?:crestodian|openclaw)\/rescue-pending\/[^"'`]*\.json\b/u, /\bcron\/(?:runs\/[^"'`]+\.jsonl|jobs\.json|jobs-state\.json)\b/u, /\b(?:process-leases|session-toggles|known-users|msteams-conversations|msteams-polls|msteams-sso-tokens|bot-storage|sync-store|thread-bindings|inbound-dedupe|startup-verification|storage-meta|crypto-idb-snapshot|command-deploy-cache|plugin-binding-approvals|plugins\/installs|config-health|port-guard|restart-sentinel|gateway-restart-intent|gateway-supervisor-restart-handoff)\.json\b/u, - /\b(?:calls|ref-index|audit\/file-transfer|audit\/openclaw)\.jsonl\b/u, + /\b(?:calls|ref-index|config-audit|audit\/(?:file-transfer|openclaw|system-agent|crestodian))\.jsonl\b/u, /\b(?:reply-cache|sent-echoes|events|claims)\.jsonl\b/u, /\bplugin-state\/state\.sqlite\b/u, /\btasks\/(?:runs\.sqlite|flows\/registry\.sqlite)\b/u, diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index d0a003e649b..16af5621b8f 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -265,9 +265,10 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +81: meeting-runtime barrel: browser meeting-bot core behind MeetingPlatformAdapter. // +3: question-gateway-runtime resolver plus request/result types. // +1: async memory prompt preparation registration. + // +1: canonical memory host event normalization for SQLite storage. // +4: gateway-backed harness question runner, claim/cancel helpers, and caller type. // Harvest: internal question runtime exports -2. - 8155, + 8156, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( @@ -303,9 +304,10 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +32: meeting-runtime barrel: browser meeting-bot core behind MeetingPlatformAdapter. // +1: question-gateway-runtime resolver. // +1: async memory prompt preparation registration. + // +1: canonical memory host event normalization for SQLite storage. // +3: gateway-backed harness question runner and claim/cancel helpers. // Harvest: internal question runtime callable -1. - 4537, + 4538, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( @@ -330,8 +332,9 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { publicWildcardReexports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_WILDCARD_REEXPORTS", // Used-union narrowing removes 103 wildcard re-exports. - // Harvest: freeze the compat config-schema barrel to explicit exports -1. - 104, + // Harvest: freeze the compat config-schema barrel to explicit exports -1; + // retire the Memory Core facade's event-store wildcard -1. + 103, env, ), }; diff --git a/src/agents/acp-parent-stream-store.sqlite.test-support.ts b/src/agents/acp-parent-stream-store.sqlite.test-support.ts new file mode 100644 index 00000000000..c8043334a69 --- /dev/null +++ b/src/agents/acp-parent-stream-store.sqlite.test-support.ts @@ -0,0 +1,25 @@ +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js"; +import { + openOpenClawAgentDatabase, + type OpenClawAgentDatabaseOptions, +} from "../state/openclaw-agent-db.js"; +import type { AcpParentStreamEvent } from "./acp-parent-stream-store.sqlite.js"; + +type AcpParentStreamDatabase = Pick; + +export function listAcpParentStreamEventsForTest( + options: OpenClawAgentDatabaseOptions & { sessionId: string; runId: string }, +): AcpParentStreamEvent[] { + const database = openOpenClawAgentDatabase(options); + const db = getNodeSqliteKysely(database.db); + return executeSqliteQuerySync( + database.db, + db + .selectFrom("acp_parent_stream_events") + .select("event_json") + .where("session_id", "=", options.sessionId) + .where("run_id", "=", options.runId) + .orderBy("seq", "asc"), + ).rows.map((row) => JSON.parse(row.event_json) as AcpParentStreamEvent); +} diff --git a/src/agents/acp-parent-stream-store.sqlite.test.ts b/src/agents/acp-parent-stream-store.sqlite.test.ts new file mode 100644 index 00000000000..cd7bd53e0f2 --- /dev/null +++ b/src/agents/acp-parent-stream-store.sqlite.test.ts @@ -0,0 +1,105 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js"; +import { + closeOpenClawAgentDatabasesForTest, + runOpenClawAgentWriteTransaction, +} from "../state/openclaw-agent-db.js"; +import { withTempDir } from "../test-helpers/temp-dir.js"; +import { recordAcpParentStreamEvents } from "./acp-parent-stream-store.sqlite.js"; +import { listAcpParentStreamEventsForTest } from "./acp-parent-stream-store.sqlite.test-support.js"; + +describe("ACP parent stream SQLite store", () => { + afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + }); + + it("orders run events and removes them with the child session", async () => { + await withTempDir({ prefix: "openclaw-acp-parent-stream-" }, async (stateDir) => { + const options = { + agentId: "codex", + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }; + runOpenClawAgentWriteTransaction((database) => { + const db = getNodeSqliteKysely>(database.db); + executeSqliteQuerySync( + database.db, + db.insertInto("sessions").values({ + session_id: "session-1", + session_key: "agent:codex:acp:child", + session_scope: "conversation", + created_at: 1, + updated_at: 1, + }), + ); + }, options); + + recordAcpParentStreamEvents({ + ...options, + sessionId: "session-1", + runId: "run-1", + events: [ + { createdAt: 10, event: { kind: "assistant_delta", delta: "one" } }, + { createdAt: 11, event: { kind: "lifecycle", phase: "end" } }, + ], + }); + + expect( + listAcpParentStreamEventsForTest({ ...options, sessionId: "session-1", runId: "run-1" }), + ).toEqual([ + { kind: "assistant_delta", delta: "one" }, + { kind: "lifecycle", phase: "end" }, + ]); + + runOpenClawAgentWriteTransaction((database) => { + const db = getNodeSqliteKysely>(database.db); + executeSqliteQuerySync( + database.db, + db.deleteFrom("sessions").where("session_id", "=", "session-1"), + ); + }, options); + expect( + listAcpParentStreamEventsForTest({ ...options, sessionId: "session-1", runId: "run-1" }), + ).toEqual([]); + }); + }); + + it("drops unserializable events without blocking later diagnostics", async () => { + await withTempDir({ prefix: "openclaw-acp-parent-stream-invalid-" }, async (stateDir) => { + const options = { + agentId: "codex", + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }; + runOpenClawAgentWriteTransaction((database) => { + const db = getNodeSqliteKysely>(database.db); + executeSqliteQuerySync( + database.db, + db.insertInto("sessions").values({ + session_id: "session-1", + session_key: "agent:codex:acp:invalid", + session_scope: "conversation", + created_at: 1, + updated_at: 1, + }), + ); + }, options); + const circular: Record = { kind: "circular" }; + circular.self = circular; + + recordAcpParentStreamEvents({ + ...options, + sessionId: "session-1", + runId: "run-1", + events: [ + { createdAt: 10, event: { toJSON: () => undefined } }, + { createdAt: 11, event: circular }, + { createdAt: 12, event: { kind: "lifecycle", phase: "end" } }, + ], + }); + + expect( + listAcpParentStreamEventsForTest({ ...options, sessionId: "session-1", runId: "run-1" }), + ).toEqual([{ kind: "lifecycle", phase: "end" }]); + }); + }); +}); diff --git a/src/agents/acp-parent-stream-store.sqlite.ts b/src/agents/acp-parent-stream-store.sqlite.ts new file mode 100644 index 00000000000..5cdd003fa3f --- /dev/null +++ b/src/agents/acp-parent-stream-store.sqlite.ts @@ -0,0 +1,77 @@ +// ACP parent-stream diagnostics live with their child session in the per-agent database. +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js"; +import { + runOpenClawAgentWriteTransaction, + type OpenClawAgentDatabaseOptions, +} from "../state/openclaw-agent-db.js"; + +type AcpParentStreamDatabase = Pick; + +export type AcpParentStreamEvent = Record; + +function getAcpParentStreamKysely(database: import("node:sqlite").DatabaseSync) { + return getNodeSqliteKysely(database); +} + +function normalizeSqliteNumber(value: number | bigint): number { + return typeof value === "bigint" ? Number(value) : value; +} + +/** Records one ordered batch in the same synchronous commit section as sequence allocation. */ +export function recordAcpParentStreamEvents( + options: OpenClawAgentDatabaseOptions & { + sessionId: string; + runId: string; + events: Array<{ event: AcpParentStreamEvent; createdAt: number }>; + }, +): void { + if (options.events.length === 0) { + return; + } + const prepared = options.events.flatMap((entry) => { + try { + const eventJson = JSON.stringify(entry.event); + if (eventJson !== undefined) { + return [{ eventJson, createdAt: entry.createdAt }]; + } + } catch { + // One malformed diagnostic must not poison later valid events or retries. + } + return []; + }); + if (prepared.length === 0) { + return; + } + runOpenClawAgentWriteTransaction((database) => { + const db = getAcpParentStreamKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("acp_parent_stream_events") + .select((eb) => eb.fn.max("seq").as("max_seq")) + .where("session_id", "=", options.sessionId) + .where("run_id", "=", options.runId), + ); + const firstSeq = + row?.max_seq === null || row?.max_seq === undefined + ? 0 + : normalizeSqliteNumber(row.max_seq) + 1; + executeSqliteQuerySync( + database.db, + db.insertInto("acp_parent_stream_events").values( + prepared.map((entry, index) => ({ + session_id: options.sessionId, + run_id: options.runId, + seq: firstSeq + index, + event_json: entry.eventJson, + created_at: entry.createdAt, + })), + ), + ); + }, options); +} diff --git a/src/agents/acp-spawn-parent-stream.test.ts b/src/agents/acp-spawn-parent-stream.test.ts index 5b3fb1bf0e6..0fca9a5747f 100644 --- a/src/agents/acp-spawn-parent-stream.test.ts +++ b/src/agents/acp-spawn-parent-stream.test.ts @@ -1,13 +1,11 @@ -/** Tests ACP child-to-parent stream relay notices, routing, and log path resolution. */ +/** Tests ACP child-to-parent stream relay notices and routing. */ import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { mergeMockedModule } from "../test-utils/vitest-module-mocks.js"; const enqueueSystemEventMock = vi.fn(); const requestHeartbeatMock = vi.fn(); -const readAcpSessionEntryMock = vi.fn(); -const resolveSessionFilePathMock = vi.fn(); -const resolveSessionFilePathOptionsMock = vi.fn(); +const recordAcpParentStreamEventsMock = vi.fn(); vi.mock("../infra/system-events.js", () => ({ enqueueSystemEvent: (...args: unknown[]) => enqueueSystemEventMock(...args), @@ -24,32 +22,18 @@ vi.mock("../infra/heartbeat-wake.js", async () => { ); }); -vi.mock("../acp/runtime/session-meta.js", async () => { +vi.mock("./acp-parent-stream-store.sqlite.js", async () => { return await mergeMockedModule( - await vi.importActual( - "../acp/runtime/session-meta.js", + await vi.importActual( + "./acp-parent-stream-store.sqlite.js", ), () => ({ - readAcpSessionEntry: (...args: unknown[]) => readAcpSessionEntryMock(...args), - }), - ); -}); - -vi.mock("../config/sessions/paths.js", async () => { - return await mergeMockedModule( - await vi.importActual( - "../config/sessions/paths.js", - ), - () => ({ - resolveSessionFilePath: (...args: unknown[]) => resolveSessionFilePathMock(...args), - resolveSessionFilePathOptions: (...args: unknown[]) => - resolveSessionFilePathOptionsMock(...args), + recordAcpParentStreamEvents: (...args: unknown[]) => recordAcpParentStreamEventsMock(...args), }), ); }); let emitAgentEvent: typeof import("../infra/agent-events.js").emitAgentEvent; -let resolveAcpSpawnStreamLogPath: typeof import("./acp-spawn-parent-stream.js").resolveAcpSpawnStreamLogPath; let startAcpSpawnParentStreamRelay: typeof import("./acp-spawn-parent-stream.js").startAcpSpawnParentStreamRelay; const progressCommentaryDeliveryContext = { @@ -103,17 +87,14 @@ function firstMockCall( describe("startAcpSpawnParentStreamRelay", () => { beforeAll(async () => { ({ emitAgentEvent } = await import("../infra/agent-events.js")); - ({ resolveAcpSpawnStreamLogPath, startAcpSpawnParentStreamRelay } = - await import("./acp-spawn-parent-stream.js")); + ({ startAcpSpawnParentStreamRelay } = await import("./acp-spawn-parent-stream.js")); }); beforeEach(() => { enqueueSystemEventMock.mockClear(); requestHeartbeatMock.mockClear(); - readAcpSessionEntryMock.mockReset(); - resolveSessionFilePathMock.mockReset(); - resolveSessionFilePathOptionsMock.mockReset(); - resolveSessionFilePathOptionsMock.mockImplementation((value: unknown) => value); + recordAcpParentStreamEventsMock.mockReset(); + recordAcpParentStreamEventsMock.mockImplementation(() => undefined); vi.useFakeTimers(); vi.setSystemTime(new Date("2026-03-04T01:00:00.000Z")); }); @@ -222,6 +203,50 @@ describe("startAcpSpawnParentStreamRelay", () => { relay.dispose(); }); + it("backs off and caps SQLite diagnostic retries", () => { + recordAcpParentStreamEventsMock + .mockImplementationOnce(() => { + throw new Error("database unavailable"); + }) + .mockImplementation(() => undefined); + const relay = startAcpSpawnParentStreamRelay({ + runId: "run-diagnostic-retry", + parentSessionKey: "agent:main:main", + childSessionKey: "agent:codex:acp:diagnostic-retry", + childSessionId: "session-diagnostic-retry", + agentId: "codex", + streamFlushMs: 120_000, + noOutputNoticeMs: 120_000, + }); + + emitAgentEvent({ + runId: "run-diagnostic-retry", + stream: "assistant", + data: { delta: "first" }, + }); + vi.advanceTimersByTime(1_000); + expect(recordAcpParentStreamEventsMock).toHaveBeenCalledTimes(1); + + for (let index = 0; index < 300; index += 1) { + emitAgentEvent({ + runId: "run-diagnostic-retry", + stream: "assistant", + data: { delta: `event-${index}` }, + }); + } + expect(recordAcpParentStreamEventsMock).toHaveBeenCalledTimes(1); + vi.advanceTimersByTime(1_999); + expect(recordAcpParentStreamEventsMock).toHaveBeenCalledTimes(1); + vi.advanceTimersByTime(1); + + expect(recordAcpParentStreamEventsMock).toHaveBeenCalledTimes(2); + const retried = recordAcpParentStreamEventsMock.mock.calls[1]?.[0] as + | { events?: unknown[] } + | undefined; + expect(retried?.events).toHaveLength(256); + relay.dispose(); + }); + it("remaps cron-run parent session keys while relaying stream events", () => { const relay = startAcpSpawnParentStreamRelay({ runId: "run-cron", @@ -1436,35 +1461,5 @@ describe("startAcpSpawnParentStreamRelay", () => { expect(collectedTexts()[1]).toBe(`codex: ${expected}`); relay.dispose(); }); - - it("resolves ACP spawn stream log path from session metadata", () => { - readAcpSessionEntryMock.mockReturnValue({ - storePath: "/tmp/openclaw/agents/codex/sessions/sessions.json", - entry: { - sessionId: "sess-123", - sessionFile: "/tmp/openclaw/agents/codex/sessions/sess-123.jsonl", - }, - }); - resolveSessionFilePathMock.mockReturnValue( - "/tmp/openclaw/agents/codex/sessions/sess-123.jsonl", - ); - - const resolved = resolveAcpSpawnStreamLogPath({ - childSessionKey: "agent:codex:acp:child-1", - }); - - expect(resolved).toBe("/tmp/openclaw/agents/codex/sessions/sess-123.acp-stream.jsonl"); - expect(readAcpSessionEntryMock).toHaveBeenCalledWith({ - sessionKey: "agent:codex:acp:child-1", - }); - expect(resolveSessionFilePathMock).toHaveBeenCalledTimes(1); - const [sessionId, entry, options] = firstMockCall( - resolveSessionFilePathMock, - "session file path resolution", - ) as [string, { sessionId?: unknown }, { storePath?: unknown }]; - expect(sessionId).toBe("sess-123"); - expect(entry.sessionId).toBe("sess-123"); - expect(options.storePath).toBe("/tmp/openclaw/agents/codex/sessions/sessions.json"); - }); }); /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/acp-spawn-parent-stream.ts b/src/agents/acp-spawn-parent-stream.ts index ece106808a1..14d06735138 100644 --- a/src/agents/acp-spawn-parent-stream.ts +++ b/src/agents/acp-spawn-parent-stream.ts @@ -1,10 +1,7 @@ /** Relays child ACP session stream updates back into the requester parent session. */ -import { mkdir } from "node:fs/promises"; -import path from "node:path"; import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { sliceUtf16Safe, truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; -import { readAcpSessionEntry } from "../acp/runtime/session-meta.js"; import { isAcpTagVisible, resolveAcpProjectionSettings, @@ -14,7 +11,6 @@ import { resolveChannelStreamingProgressCommentary, type StreamingCompatEntry, } from "../channels/streaming.js"; -import { resolveSessionFilePath, resolveSessionFilePathOptions } from "../config/sessions/paths.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { onAgentEvent } from "../infra/agent-events.js"; import { @@ -23,13 +19,17 @@ import { scopedHeartbeatWakeOptionsForPolicy, } from "../infra/event-session-routing.js"; import { requestHeartbeat } from "../infra/heartbeat-wake.js"; -import { appendRegularFile } from "../infra/regular-file.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; import { resolveNormalizedAccountEntry } from "../routing/account-lookup.js"; import { normalizeAccountId } from "../routing/session-key.js"; import { normalizeAssistantPhase } from "../shared/chat-message-content.js"; import { recordTaskRunProgressByRunId } from "../tasks/detached-task-runtime.js"; import type { DeliveryContext } from "../utils/delivery-context.types.js"; +import { + recordAcpParentStreamEvents, + type AcpParentStreamEvent, +} from "./acp-parent-stream-store.sqlite.js"; const DEFAULT_STREAM_FLUSH_MS = 2_500; const DEFAULT_NO_OUTPUT_NOTICE_MS = 60_000; @@ -37,6 +37,11 @@ const DEFAULT_NO_OUTPUT_POLL_MS = 15_000; const DEFAULT_MAX_RELAY_LIFETIME_MS = 6 * 60 * 60 * 1000; const STREAM_BUFFER_MAX_CHARS = 4_000; const STREAM_SNIPPET_MAX_CHARS = 220; +const STREAM_LOG_BATCH_SIZE = 100; +const STREAM_LOG_FLUSH_MS = 1_000; +const STREAM_LOG_MAX_PENDING_EVENTS = 256; +const STREAM_LOG_MAX_RETRY_MS = 30_000; +const log = createSubsystemLogger("agents/acp-parent-stream"); type AcpParentProgressStreamingConfig = StreamingCompatEntry & { accounts?: Record; @@ -206,46 +211,14 @@ function shouldRelayAcpStatusProgress(params: { return isAcpTagVisible(params.projectionSettings, params.tag); } -function resolveAcpStreamLogPathFromSessionFile(sessionFile: string, sessionId: string): string { - const baseDir = path.dirname(path.resolve(sessionFile)); - return path.join(baseDir, `${sessionId}.acp-stream.jsonl`); -} - -/** Resolves the JSONL stream log path for an ACP child session when metadata exists. */ -export function resolveAcpSpawnStreamLogPath(params: { - childSessionKey: string; -}): string | undefined { - const childSessionKey = normalizeOptionalString(params.childSessionKey); - if (!childSessionKey) { - return undefined; - } - const storeEntry = readAcpSessionEntry({ - sessionKey: childSessionKey, - }); - const sessionId = normalizeOptionalString(storeEntry?.entry?.sessionId); - if (!storeEntry || !sessionId) { - return undefined; - } - try { - const sessionFile = resolveSessionFilePath( - sessionId, - storeEntry.entry, - resolveSessionFilePathOptions({ - storePath: storeEntry.storePath, - }), - ); - return resolveAcpStreamLogPathFromSessionFile(sessionFile, sessionId); - } catch { - return undefined; - } -} - /** Starts a bounded parent-session relay for child ACP output and progress notices. */ export function startAcpSpawnParentStreamRelay(params: { runId: string; parentSessionKey: string; childSessionKey: string; + childSessionId?: string; agentId: string; + env?: NodeJS.ProcessEnv; /** * Optional `session.mainKey` from the runtime config. Used to remap * cron-run parent session keys to the agent's main queue when relaying @@ -262,7 +235,6 @@ export function startAcpSpawnParentStreamRelay(params: { */ sessionScope?: "per-sender" | "global"; eventRouting?: EventSessionRoutingPolicy; - logPath?: string; deliveryContext?: DeliveryContext; surfaceUpdates?: boolean; streamFlushMs?: number; @@ -300,67 +272,105 @@ export function startAcpSpawnParentStreamRelay(params: { const relayLabel = truncate(compactWhitespace(params.agentId), 40) || "ACP child"; const contextPrefix = `acp-spawn:${runId}`; - const logPath = normalizeOptionalString(params.logPath); - let logDirReady = false; - let pendingLogLines = ""; - let logFlushScheduled = false; - let logWriteChain: Promise = Promise.resolve(); - const flushLogBuffer = () => { - if (!logPath || !pendingLogLines) { + const childSessionId = normalizeOptionalString(params.childSessionId); + // Delayed flushes must keep the state database selected when the relay started. + const stateEnv = { ...(params.env ?? process.env) }; + const pendingLogEvents: Array<{ event: AcpParentStreamEvent; createdAt: number }> = []; + let logFlushTimer: NodeJS.Timeout | undefined; + let logFailureWarned = false; + let logBufferWarned = false; + let consecutiveLogFailures = 0; + let disposed = false; + const capPendingLogEvents = () => { + const overflow = pendingLogEvents.length - STREAM_LOG_MAX_PENDING_EVENTS; + if (overflow <= 0) { return; } - const chunk = pendingLogLines; - pendingLogLines = ""; - logWriteChain = logWriteChain - .then(async () => { - if (!logDirReady) { - await mkdir(path.dirname(logPath), { - recursive: true, - }); - logDirReady = true; - } - await appendRegularFile({ filePath: logPath, content: chunk }); - }) - .catch(() => { - // Best-effort diagnostics; never break relay flow. + pendingLogEvents.splice(0, overflow); + if (!logBufferWarned) { + log.warn("Capped ACP parent stream diagnostic buffer", { + runId, + childSessionId, + maxPendingEvents: STREAM_LOG_MAX_PENDING_EVENTS, }); + logBufferWarned = true; + } }; - const scheduleLogFlush = () => { - if (!logPath || logFlushScheduled) { + const clearLogFlushTimer = () => { + if (!logFlushTimer) { return; } - logFlushScheduled = true; - queueMicrotask(() => { - logFlushScheduled = false; - flushLogBuffer(); - }); + clearTimeout(logFlushTimer); + logFlushTimer = undefined; }; - const writeLogLine = (entry: Record) => { - if (!logPath) { + function flushLogEvents(options: { terminal?: boolean } = {}) { + clearLogFlushTimer(); + if (!childSessionId || pendingLogEvents.length === 0) { return; } + const events = pendingLogEvents.splice(0); try { - pendingLogLines += `${JSON.stringify(entry)}\n`; - if (pendingLogLines.length >= 16_384) { - flushLogBuffer(); - return; + recordAcpParentStreamEvents({ + agentId: params.agentId, + env: stateEnv, + sessionId: childSessionId, + runId, + events, + }); + logFailureWarned = false; + logBufferWarned = false; + consecutiveLogFailures = 0; + } catch (error) { + if (!options.terminal) { + pendingLogEvents.unshift(...events); + capPendingLogEvents(); + consecutiveLogFailures += 1; + scheduleLogFlush( + Math.min(STREAM_LOG_FLUSH_MS * 2 ** consecutiveLogFailures, STREAM_LOG_MAX_RETRY_MS), + ); + } + if (!logFailureWarned || options.terminal) { + log.warn("Failed to persist ACP parent stream diagnostics", { + runId, + childSessionId, + retrying: !options.terminal, + error: String(error), + }); + logFailureWarned = true; } - scheduleLogFlush(); - } catch { - // Best-effort diagnostics; never break relay flow. } - }; + } + function scheduleLogFlush(delayMs = STREAM_LOG_FLUSH_MS) { + if (disposed || logFlushTimer || pendingLogEvents.length === 0) { + return; + } + logFlushTimer = setTimeout(() => flushLogEvents(), delayMs); + logFlushTimer.unref?.(); + } const logEvent = (kind: string, fields?: Record) => { - writeLogLine({ - ts: new Date().toISOString(), - epochMs: Date.now(), - runId, - parentSessionKey, - childSessionKey: params.childSessionKey, - agentId: params.agentId, - kind, - ...fields, + if (!childSessionId) { + return; + } + const createdAt = Date.now(); + pendingLogEvents.push({ + createdAt, + event: { + ts: new Date(createdAt).toISOString(), + epochMs: createdAt, + runId, + parentSessionKey, + childSessionKey: params.childSessionKey, + agentId: params.agentId, + kind, + ...fields, + }, }); + capPendingLogEvents(); + if (consecutiveLogFailures === 0 && pendingLogEvents.length >= STREAM_LOG_BATCH_SIZE) { + flushLogEvents(); + return; + } + scheduleLogFlush(); }; const shouldSurfaceUpdates = params.surfaceUpdates !== false; const shouldRelayProgressCommentary = resolveParentProgressCommentary({ @@ -418,7 +428,6 @@ export function startAcpSpawnParentStreamRelay(params: { ); }; - let disposed = false; let pendingText = ""; let pendingProgressKind: string | undefined; let replaceableAssistantSnapshot: string | undefined; @@ -757,8 +766,8 @@ export function startAcpSpawnParentStreamRelay(params: { } disposed = true; clearFlushTimer(); + flushLogEvents({ terminal: true }); clearRelayLifetimeTimer(); - flushLogBuffer(); clearInterval(noOutputWatcherTimer); unsubscribe(); }; diff --git a/src/agents/acp-spawn.test.ts b/src/agents/acp-spawn.test.ts index 9d9bb63a4ce..63a6eea7933 100644 --- a/src/agents/acp-spawn.test.ts +++ b/src/agents/acp-spawn.test.ts @@ -58,7 +58,6 @@ const hoisted = vi.hoisted(() => { const initializeSessionMock = vi.fn(); const getAcpSessionManagerMock = vi.fn(); const startAcpSpawnParentStreamRelayMock = vi.fn(); - const resolveAcpSpawnStreamLogPathMock = vi.fn(); const loadSessionStoreMock = vi.fn(); const readAcpSessionMetaMock = vi.fn(); const resolveStorePathMock = vi.fn(); @@ -150,7 +149,6 @@ const hoisted = vi.hoisted(() => { initializeSessionMock, getAcpSessionManagerMock, startAcpSpawnParentStreamRelayMock, - resolveAcpSpawnStreamLogPathMock, loadSessionStoreMock, readAcpSessionMetaMock, resolveStorePathMock, @@ -229,7 +227,6 @@ vi.mock("../tasks/detached-task-runtime.js", () => ({ })); vi.mock("./acp-spawn-parent-stream.js", () => ({ - resolveAcpSpawnStreamLogPath: hoisted.resolveAcpSpawnStreamLogPathMock, startAcpSpawnParentStreamRelay: hoisted.startAcpSpawnParentStreamRelayMock, })); @@ -811,9 +808,6 @@ describe("spawnAcpDirect", () => { hoisted.startAcpSpawnParentStreamRelayMock .mockReset() .mockImplementation(() => createRelayHandle()); - hoisted.resolveAcpSpawnStreamLogPathMock - .mockReset() - .mockReturnValue("/tmp/sess-main.acp-stream.jsonl"); hoisted.resolveStorePathMock.mockReset().mockReturnValue("/tmp/codex-sessions.json"); hoisted.readAcpSessionMetaMock.mockReset().mockReturnValue(undefined); hoisted.loadSessionStoreMock.mockReset().mockImplementation(() => { @@ -2424,7 +2418,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); if (expectTranscriptPersistence) { expectRecordFields( @@ -2625,7 +2618,6 @@ describe("spawnAcpDirect", () => { ); const accepted = expectAcceptedSpawn(result); - expect(accepted.streamLogPath).toBe("/tmp/sess-main.acp-stream.jsonl"); const agentCall = hoisted.callGatewayMock.mock.calls .map((call: unknown[]) => call[0] as { method?: string; params?: Record }) .find((request) => request.method === "agent"); @@ -2647,7 +2639,7 @@ describe("spawnAcpDirect", () => { expectRelayCallFields({ parentSessionKey: "agent:main:main", agentId: "codex", - logPath: "/tmp/sess-main.acp-stream.jsonl", + childSessionId: "sess-123", emitStartNotice: false, }); const relayRuns = hoisted.startAcpSpawnParentStreamRelayMock.mock.calls.map( @@ -2655,11 +2647,6 @@ describe("spawnAcpDirect", () => { ); expect(relayRuns).toContain(agentCall?.params?.idempotencyKey); expect(relayRuns).toContain(accepted.runId); - const streamPathInput = expectRecordFields( - firstMockCall(hoisted.resolveAcpSpawnStreamLogPathMock, "stream log path resolution")[0], - {}, - ); - expect(streamPathInput.childSessionKey).toMatch(/^agent:codex:acp:/); expect(firstHandle.dispose).toHaveBeenCalledTimes(1); expect(firstHandle.notifyStarted).not.toHaveBeenCalled(); expect(secondHandle.notifyStarted).toHaveBeenCalledTimes(1); @@ -2724,7 +2711,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBe("/tmp/sess-main.acp-stream.jsonl"); const agentCall = hoisted.callGatewayMock.mock.calls .map((call: unknown[]) => call[0] as { method?: string; params?: Record }) .find((request) => request.method === "agent"); @@ -2735,7 +2721,7 @@ describe("spawnAcpDirect", () => { expectRelayCallFields({ parentSessionKey: "agent:main:subagent:parent", agentId: "codex", - logPath: "/tmp/sess-main.acp-stream.jsonl", + childSessionId: "sess-123", deliveryContext: { channel: "discord", to: "channel:parent-channel", @@ -2812,7 +2798,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); }); @@ -2843,7 +2828,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); }); @@ -2877,7 +2861,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); }); @@ -2933,7 +2916,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); }); @@ -2963,7 +2945,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); }); @@ -2982,7 +2963,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); }); @@ -2999,7 +2979,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); }); @@ -3020,7 +2999,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); }); @@ -3053,7 +3031,6 @@ describe("spawnAcpDirect", () => { const accepted = expectAcceptedSpawn(result); expect(accepted.mode).toBe("run"); - expect(accepted.streamLogPath).toBeUndefined(); expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled(); }); diff --git a/src/agents/acp-spawn.ts b/src/agents/acp-spawn.ts index 0d1172e3d33..5157c6f48f5 100644 --- a/src/agents/acp-spawn.ts +++ b/src/agents/acp-spawn.ts @@ -77,7 +77,6 @@ import { listTasksForOwnerKey } from "../tasks/runtime-internal.js"; import { deliveryContextFromSession, normalizeDeliveryContext } from "../utils/delivery-context.js"; import { type AcpSpawnParentRelayHandle, - resolveAcpSpawnStreamLogPath, startAcpSpawnParentStreamRelay, } from "./acp-spawn-parent-stream.js"; import { listAgentIds, resolveAgentConfig, resolveDefaultAgentId } from "./agent-scope.js"; @@ -206,7 +205,6 @@ type SpawnAcpResultFields = { mode?: SpawnAcpMode; runTimeoutSeconds?: number; inlineDelivery?: boolean; - streamLogPath?: string; note?: string; }; @@ -1478,6 +1476,7 @@ export async function spawnAcpDirect( let binding: SessionBindingRecord | null = null; let sessionCreated = false; + let childSessionId: string | undefined; let initializedRuntime: AcpSpawnRuntimeCloseHandle | undefined; try { await callGateway({ @@ -1504,6 +1503,7 @@ export async function spawnAcpDirect( cwd: runtimeCwd, }); initializedRuntime = initializedSession.runtimeCloseHandle; + childSessionId = initializedSession.sessionId; if (preparedBinding) { ({ binding } = await bindPreparedAcpThread({ @@ -1548,12 +1548,6 @@ export async function spawnAcpDirect( requesterSessionKey: requesterInternalKey, agentId: targetAgentId, }); - const streamLogPath = - effectiveStreamToParent && parentSessionKey - ? resolveAcpSpawnStreamLogPath({ - childSessionKey: sessionKey, - }) - : undefined; const parentAgentId = parentSessionKey ? resolveAgentIdFromSessionKey(parentSessionKey) : undefined; @@ -1571,6 +1565,7 @@ export async function spawnAcpDirect( : undefined; let parentRelay: AcpSpawnParentRelayHandle | undefined; + const parentRelayStateEnv = { ...process.env }; const parentEventRouting = parentSessionKey ? resolveEventSessionRoutingPolicy({ cfg, sessionKey: parentSessionKey }) : undefined; @@ -1580,11 +1575,12 @@ export async function spawnAcpDirect( runId: childIdem, parentSessionKey, childSessionKey: sessionKey, + childSessionId, agentId: targetAgentId, + env: parentRelayStateEnv, mainKey: cfg.session?.mainKey, sessionScope: cfg.session?.scope, eventRouting: parentEventRouting, - logPath: streamLogPath, deliveryContext: parentDeliveryCtx, emitStartNotice: false, cfg, @@ -1640,11 +1636,12 @@ export async function spawnAcpDirect( runId: childRunId, parentSessionKey, childSessionKey: sessionKey, + childSessionId, agentId: targetAgentId, + env: parentRelayStateEnv, mainKey: cfg.session?.mainKey, sessionScope: cfg.session?.scope, eventRouting: parentEventRouting, - logPath: streamLogPath, deliveryContext: parentDeliveryCtx, emitStartNotice: false, cfg, @@ -1687,7 +1684,6 @@ export async function spawnAcpDirect( runId: childRunId, mode: spawnMode, runTimeoutSeconds, - ...(streamLogPath ? { streamLogPath } : {}), note: spawnMode === "session" ? ACP_SPAWN_SESSION_ACCEPTED_NOTE : ACP_SPAWN_ACCEPTED_NOTE, }; } diff --git a/src/cli/gateway-cli/run.option-collisions.test.ts b/src/cli/gateway-cli/run.option-collisions.test.ts index 8f191b242aa..e48989ff5db 100644 --- a/src/cli/gateway-cli/run.option-collisions.test.ts +++ b/src/cli/gateway-cli/run.option-collisions.test.ts @@ -1,7 +1,7 @@ // Gateway run option collision tests cover gateway run flag registration boundaries. -import path from "node:path"; import { Command } from "commander"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { CONFIG_AUDIT_STORE_LABEL } from "../../config/io.audit.js"; import type { ConfigFileSnapshot } from "../../config/types.js"; import { GATEWAY_SERVICE_RUNTIME_PID_ENV } from "../../daemon/constants.js"; import { SUPERVISOR_HINT_ENV_VARS } from "../../infra/supervisor-markers.js"; @@ -1589,9 +1589,7 @@ describe("gateway run option collisions", () => { expect(runtimeErrors).toContain( "Gateway start blocked: existing config is missing gateway.mode. Treat this as suspicious or clobbered config. Re-run `openclaw onboard --mode local` or `openclaw setup`, set gateway.mode=local manually, or pass --allow-unconfigured.", ); - expect(runtimeErrors).toContain( - `Config write audit: ${path.join("/tmp", "logs", "config-audit.jsonl")}`, - ); + expect(runtimeErrors).toContain(`Config write audit: ${CONFIG_AUDIT_STORE_LABEL}`); expect(startGatewayServer).not.toHaveBeenCalled(); expect(readBestEffortConfig).not.toHaveBeenCalled(); }); @@ -1613,9 +1611,7 @@ describe("gateway run option collisions", () => { expect(runtimeErrors).toContain( "Gateway start blocked: existing config is missing gateway.mode. Treat this as suspicious or clobbered config. Re-run `openclaw onboard --mode local` or `openclaw setup`, set gateway.mode=local manually, or pass --allow-unconfigured.", ); - expect(runtimeErrors).toContain( - `Config write audit: ${path.join("/tmp", "logs", "config-audit.jsonl")}`, - ); + expect(runtimeErrors).toContain(`Config write audit: ${CONFIG_AUDIT_STORE_LABEL}`); expect(readConfigFileSnapshotWithPluginMetadata).toHaveBeenCalledOnce(); expect(startGatewayServer).not.toHaveBeenCalled(); }); diff --git a/src/cli/gateway-cli/run.ts b/src/cli/gateway-cli/run.ts index d8804b749a6..93cc51bfaea 100644 --- a/src/cli/gateway-cli/run.ts +++ b/src/cli/gateway-cli/run.ts @@ -2,7 +2,6 @@ import fs from "node:fs"; import { request as httpRequest } from "node:http"; import { request as httpsRequest } from "node:https"; -import path from "node:path"; import { TLSSocket } from "node:tls"; import { expectDefined } from "@openclaw/normalization-core"; import { @@ -17,16 +16,12 @@ import type { ReadConfigFileSnapshotWithPluginMetadataResult, } from "../../config/config.js"; import { ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS_ENV } from "../../config/future-version-guard.js"; +import { CONFIG_AUDIT_STORE_LABEL } from "../../config/io.audit.js"; import { isDoctorRecoverableInvalidConfigError, isInvalidConfigError, } from "../../config/io.invalid-config.js"; -import { - CONFIG_PATH, - normalizeStateDirEnv, - resolveGatewayPort, - resolveStateDir, -} from "../../config/paths.js"; +import { CONFIG_PATH, normalizeStateDirEnv, resolveGatewayPort } from "../../config/paths.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { hasConfiguredSecretInput } from "../../config/types.secrets.js"; import { GATEWAY_SERVICE_RUNTIME_PID_ENV } from "../../daemon/constants.js"; @@ -250,7 +245,7 @@ function shouldBlockGatewayBindWithoutExplicitAuth(params: { function getGatewayStartGuardErrors(params: { allowUnconfigured?: boolean; configExists: boolean; - configAuditPath: string; + configAuditLocation: string; mode: string | undefined; }): string[] { if (params.allowUnconfigured || params.mode === "local") { @@ -268,12 +263,12 @@ function getGatewayStartGuardErrors(params: { "Treat this as suspicious or clobbered config.", `Re-run \`${formatCliCommand("openclaw onboard --mode local")}\` or \`${formatCliCommand("openclaw setup")}\`, set gateway.mode=local manually, or pass --allow-unconfigured.`, ].join(" "), - `Config write audit: ${params.configAuditPath}`, + `Config write audit: ${params.configAuditLocation}`, ]; } return [ `Gateway start blocked: set gateway.mode=local (current: ${params.mode}) or pass --allow-unconfigured.`, - `Config write audit: ${params.configAuditPath}`, + `Config write audit: ${params.configAuditLocation}`, ]; } @@ -959,13 +954,12 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt gatewayLog.info("resolving authentication…"); const configExists = snapshot?.exists ?? fs.existsSync(CONFIG_PATH); - const configAuditPath = path.join(resolveStateDir(process.env), "logs", "config-audit.jsonl"); const effectiveCfg = snapshot?.valid ? snapshot.config : cfg; const mode = effectiveCfg.gateway?.mode; const guardErrors = getGatewayStartGuardErrors({ allowUnconfigured: opts.allowUnconfigured, configExists, - configAuditPath, + configAuditLocation: CONFIG_AUDIT_STORE_LABEL, mode, }); if (guardErrors.length > 0) { diff --git a/src/commands/doctor-config-audit-scrub.ts b/src/commands/doctor-config-audit-scrub.ts index 915820e1432..50df942aff4 100644 --- a/src/commands/doctor-config-audit-scrub.ts +++ b/src/commands/doctor-config-audit-scrub.ts @@ -3,7 +3,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import { note } from "../../packages/terminal-core/src/note.js"; import { - resolveConfigAuditLogPath, + resolveLegacyConfigAuditLogPath, scrubConfigAuditLog, type ConfigAuditScrubResult, } from "../config/io.audit.js"; @@ -30,7 +30,7 @@ export async function detectConfigAuditScrubIssue(params?: { }); return { ...result, - auditPath: resolveConfigAuditLogPath(env, homedir), + auditPath: resolveLegacyConfigAuditLogPath(env, homedir), }; } diff --git a/src/commands/doctor.e2e-harness.ts b/src/commands/doctor.e2e-harness.ts index 3aa697e5718..8a571fb8e4f 100644 --- a/src/commands/doctor.e2e-harness.ts +++ b/src/commands/doctor.e2e-harness.ts @@ -289,6 +289,10 @@ function createLegacyStateMigrationDetectionResult(params?: { sourcePath: "/tmp/state/commitments/commitments.json", hasLegacy: false, }, + auditLogs: { + sources: [], + hasLegacy: false, + }, acpReplayLedger: { sourcePath: "/tmp/state/acp/event-ledger.json", hasLegacy: false, diff --git a/src/config/io.audit.test-support.ts b/src/config/io.audit.test-support.ts new file mode 100644 index 00000000000..7cdad42648d --- /dev/null +++ b/src/config/io.audit.test-support.ts @@ -0,0 +1,23 @@ +import { createSqliteAuditRecordStore } from "../infra/sqlite-audit-record-store.js"; +import { + CONFIG_AUDIT_MAX_ENTRIES, + CONFIG_AUDIT_SCOPE, + type ConfigAuditRecord, +} from "./io.audit.js"; +import { resolveStateDir } from "./paths.js"; + +export function listConfigAuditRecordsForTests(params: { + env: NodeJS.ProcessEnv; + homedir: () => string; +}): ConfigAuditRecord[] { + return createSqliteAuditRecordStore({ + scope: CONFIG_AUDIT_SCOPE, + maxEntries: CONFIG_AUDIT_MAX_ENTRIES, + env: { + ...params.env, + OPENCLAW_STATE_DIR: resolveStateDir(params.env, params.homedir), + }, + }) + .entries() + .map((entry) => entry.value); +} diff --git a/src/config/io.audit.test.ts b/src/config/io.audit.test.ts index cfd7738390c..36693322f57 100644 --- a/src/config/io.audit.test.ts +++ b/src/config/io.audit.test.ts @@ -2,16 +2,18 @@ import fs from "node:fs"; import { promises as fsPromises } from "node:fs"; import path from "node:path"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; import { appendConfigAuditRecord, createConfigWriteAuditRecordBase, finalizeConfigWriteAuditRecord, formatConfigOverwriteLogMessage, - resolveConfigAuditLogPath, + resolveLegacyConfigAuditLogPath, scrubConfigAuditLog, } from "./io.audit.js"; +import { listConfigAuditRecordsForTests } from "./io.audit.test-support.js"; function createAuditRecordBase(configPath: string, argv?: string[]) { return createConfigWriteAuditRecordBase({ @@ -66,7 +68,7 @@ function createRenameAuditRecord(home: string) { }); } -function readAuditLog(home: string): unknown[] { +function readLegacyAuditLog(home: string): unknown[] { const auditPath = path.join(home, ".openclaw", "logs", "config-audit.jsonl"); return fs .readFileSync(auditPath, "utf-8") @@ -93,9 +95,13 @@ describe("config io audit helpers", () => { await suiteRootTracker.cleanup(); }); + afterEach(() => { + resetPluginStateStoreForTests(); + }); + it('ignores literal "undefined" home env values when choosing the audit log path', async () => { const home = await suiteRootTracker.make("home"); - const auditPath = resolveConfigAuditLogPath( + const auditPath = resolveLegacyConfigAuditLogPath( { HOME: "undefined", USERPROFILE: "null", @@ -196,18 +202,20 @@ describe("config io audit helpers", () => { expect(record.errorMessage).toBe("disk full"); }); - it("appends JSONL audit entries to the resolved audit path", async () => { + it("appends audit entries to shared SQLite state", async () => { const home = await suiteRootTracker.make("append"); const record = createRenameAuditRecord(home); await appendConfigAuditRecord({ - fs, env: {} as NodeJS.ProcessEnv, homedir: () => home, record, }); - const records = readAuditLog(home); + const records = listConfigAuditRecordsForTests({ + env: {} as NodeJS.ProcessEnv, + homedir: () => home, + }); expect(records).toHaveLength(1); const written = requireAuditRecord(records[0]); expect(written.event).toBe("config.write"); @@ -230,15 +238,16 @@ describe("config io audit helpers", () => { }); await appendConfigAuditRecord({ - fs, env: {} as NodeJS.ProcessEnv, homedir: () => home, record, }); - const raw = fs.readFileSync( - path.join(home, ".openclaw", "logs", "config-audit.jsonl"), - "utf-8", + const raw = JSON.stringify( + listConfigAuditRecordsForTests({ + env: {} as NodeJS.ProcessEnv, + homedir: () => home, + }), ); expect(raw).not.toContain("AIzaSyD-very-real-looking"); expect(raw).not.toContain("ya29.fake-access-token"); @@ -355,6 +364,99 @@ describe("config io audit helpers", () => { argv: ["openclaw", "--token"], expected: ["openclaw", "--token"], }, + { + name: "sensitive config set positional value", + argv: ["openclaw", "config", "set", "channels.slack.token", "secret-value"], + expected: ["openclaw", "config", "set", "channels.slack.token", "***"], + }, + { + name: "sensitive config set value after boolean option", + argv: ["openclaw", "config", "set", "--json", "channels.slack.token", '"secret-value"'], + expected: ["openclaw", "config", "set", "--json", "channels.slack.token", "***"], + }, + { + name: "sensitive config set value after root value option", + argv: [ + "openclaw", + "config", + "set", + "--profile", + "work", + "channels.slack.token", + "secret-value", + ], + expected: ["openclaw", "config", "set", "--profile", "work", "channels.slack.token", "***"], + }, + { + name: "sensitive config set value after option before subcommand", + argv: [ + "openclaw", + "config", + "--profile", + "work", + "set", + "channels.slack.token", + "secret-value", + ], + expected: ["openclaw", "config", "--profile", "work", "set", "channels.slack.token", "***"], + }, + { + name: "sensitive config set value after config parent option", + argv: [ + "openclaw", + "config", + "--section", + "channels", + "set", + "channels.slack.token", + "secret-value", + ], + expected: [ + "openclaw", + "config", + "--section", + "channels", + "set", + "channels.slack.token", + "***", + ], + }, + { + name: "sensitive config set value when a root option value is config", + argv: [ + "openclaw", + "--profile", + "config", + "config", + "set", + "channels.slack.token", + "secret-value", + ], + expected: ["openclaw", "--profile", "config", "config", "set", "channels.slack.token", "***"], + }, + { + name: "sensitive config set value after interleaved option", + argv: [ + "openclaw", + "config", + "set", + "channels.slack.token", + "--strict-json", + '"secret-value"', + ], + expected: ["openclaw", "config", "set", "channels.slack.token", "--strict-json", "***"], + }, + { + name: "config set batch JSON", + argv: [ + "openclaw", + "config", + "set", + "--batch-json", + '[{"path":"channels.slack.token","value":"secret-value"}]', + ], + expected: ["openclaw", "config", "set", "--batch-json", "***"], + }, ])("redacts $name in persisted audit process info", ({ argv, expected }) => { expect(createAuditRecordBase("/tmp/openclaw.json", argv).argv).toEqual(expected); }); @@ -364,13 +466,15 @@ describe("config io audit helpers", () => { const record = createRenameAuditRecord(home); await appendConfigAuditRecord({ - fs, env: {} as NodeJS.ProcessEnv, homedir: () => home, ...record, }); - const records = readAuditLog(home); + const records = listConfigAuditRecordsForTests({ + env: {} as NodeJS.ProcessEnv, + homedir: () => home, + }); expect(records).toHaveLength(1); const written = requireAuditRecord(records[0]); expect(written.event).toBe("config.write"); @@ -429,7 +533,7 @@ describe("config io audit helpers", () => { }); expect(result).toEqual({ scanned: 2, rewritten: 1, skipped: 0, aborted: false }); - const after = readAuditLog(home); + const after = readLegacyAuditLog(home); expect(after).toHaveLength(2); const firstAfter = requireAuditRecord(after[0]); const secondAfter = requireAuditRecord(after[1]); diff --git a/src/config/io.audit.ts b/src/config/io.audit.ts index ffb49503d88..594beb29e3d 100644 --- a/src/config/io.audit.ts +++ b/src/config/io.audit.ts @@ -1,13 +1,127 @@ // Audits config paths and values for diagnostics and safety checks. +import { randomUUID } from "node:crypto"; import path from "node:path"; +import { createSqliteAuditRecordStore } from "../infra/sqlite-audit-record-store.js"; import { redactSecrets } from "../logging/redact.js"; import { resolveStateDir } from "./paths.js"; import { redactSensitiveArgv } from "./redact-argv.js"; +import { isSensitiveConfigPath } from "./sensitive-paths.js"; const CONFIG_AUDIT_ARGV_CAP = 8; +const CONFIG_SET_VALUE_OPTIONS = new Set([ + "--batch-file", + "--batch-json", + "--container", + "--log-level", + "--profile", + "--provider-allowlist", + "--provider-arg", + "--provider-command", + "--provider-env", + "--provider-max-bytes", + "--provider-max-output-bytes", + "--provider-mode", + "--provider-no-output-timeout-ms", + "--provider-pass-env", + "--provider-path", + "--provider-source", + "--provider-timeout-ms", + "--provider-trusted-dir", + "--ref-id", + "--ref-provider", + "--ref-source", + "--section", +]); + +function findConfigSetPositionals(argv: readonly string[], setIndex: number): number[] { + const positionals: number[] = []; + let optionsEnded = false; + for (let index = setIndex + 1; index < argv.length && positionals.length < 2; index += 1) { + const arg = argv[index]; + if (arg === undefined) { + break; + } + if (!optionsEnded && arg === "--") { + optionsEnded = true; + continue; + } + if (!optionsEnded && arg.startsWith("-")) { + const equalsIndex = arg.indexOf("="); + const optionName = equalsIndex < 0 ? arg : arg.slice(0, equalsIndex); + if (equalsIndex < 0 && CONFIG_SET_VALUE_OPTIONS.has(optionName)) { + index += 1; + } + continue; + } + positionals.push(index); + } + return positionals; +} + +function findConfigSetCommandIndex(argv: readonly string[], configIndex: number): number { + let optionsEnded = false; + for (let index = configIndex + 1; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === undefined) { + return -1; + } + if (!optionsEnded && arg === "--") { + optionsEnded = true; + continue; + } + if (!optionsEnded && arg.startsWith("-")) { + const equalsIndex = arg.indexOf("="); + const optionName = equalsIndex < 0 ? arg : arg.slice(0, equalsIndex); + if (equalsIndex < 0 && CONFIG_SET_VALUE_OPTIONS.has(optionName)) { + index += 1; + } + continue; + } + return arg === "set" ? index : -1; + } + return -1; +} function redactConfigAuditArgv(argv: readonly string[]): string[] { - return redactSensitiveArgv(argv); + const redacted = redactSensitiveArgv(argv); + let setIndex = -1; + for (let index = 0; index < redacted.length; index += 1) { + if (redacted[index] !== "config") { + continue; + } + setIndex = findConfigSetCommandIndex(redacted, index); + if (setIndex >= 0) { + break; + } + } + if (setIndex < 0) { + return redacted; + } + for (let index = setIndex + 1; index < redacted.length; index += 1) { + const arg = redacted[index]; + if (arg === undefined) { + break; + } + if (arg === "--batch-json" && index + 1 < redacted.length) { + redacted[index + 1] = "***"; + index += 1; + continue; + } + if (arg.startsWith("--batch-json=")) { + redacted[index] = "--batch-json=***"; + } + } + const positionals = findConfigSetPositionals(redacted, setIndex); + if (positionals.length < 2) { + return redacted; + } + const pathIndex = positionals[0]!; + const valueIndex = positionals[1]!; + const configPath = redacted[pathIndex]; + if (typeof configPath === "string" && isSensitiveConfigPath(configPath)) { + redacted[valueIndex] = "***"; + } + return redacted; } function capArgv(argv: readonly string[] | undefined): string[] { @@ -27,7 +141,11 @@ export function snapshotConfigAuditProcessInfo(): ConfigAuditProcessInfo { }; } -const CONFIG_AUDIT_LOG_FILENAME = "config-audit.jsonl"; +export const CONFIG_AUDIT_SCOPE = "config-audit"; +export const CONFIG_AUDIT_MAX_ENTRIES = 50_000; +export const CONFIG_AUDIT_STORE_LABEL = + "SQLite diagnostic_events/config-audit state (latest 50000 rows)"; +const LEGACY_CONFIG_AUDIT_LOG_FILENAME = ["config-audit", "jsonl"].join("."); export type ConfigWriteAuditResult = "rename" | "copy-fallback" | "failed" | "rejected"; @@ -127,7 +245,7 @@ export type ConfigObserveAuditRecord = { restoreErrorMessage: string | null; }; -type ConfigAuditRecord = ConfigWriteAuditRecord | ConfigObserveAuditRecord; +export type ConfigAuditRecord = ConfigWriteAuditRecord | ConfigObserveAuditRecord; type ConfigAuditStatMetadata = { dev: string | null; @@ -162,23 +280,6 @@ type ConfigWriteAuditRecordBase = Omit< nextBytes: number; }; -type ConfigAuditFs = { - promises: { - mkdir(path: string, options?: { recursive?: boolean; mode?: number }): Promise; - appendFile( - path: string, - data: string, - options?: { encoding?: BufferEncoding; mode?: number }, - ): Promise; - }; - mkdirSync(path: string, options?: { recursive?: boolean; mode?: number }): unknown; - appendFileSync( - path: string, - data: string, - options?: { encoding?: BufferEncoding; mode?: number }, - ): unknown; -}; - function normalizeAuditLabel(value: string | undefined): string | null { if (typeof value !== "string") { return null; @@ -200,8 +301,11 @@ function resolveConfigAuditProcessInfo( return snapshotConfigAuditProcessInfo(); } -export function resolveConfigAuditLogPath(env: NodeJS.ProcessEnv, homedir: () => string): string { - return path.join(resolveStateDir(env, homedir), "logs", CONFIG_AUDIT_LOG_FILENAME); +export function resolveLegacyConfigAuditLogPath( + env: NodeJS.ProcessEnv, + homedir: () => string, +): string { + return path.join(resolveStateDir(env, homedir), "logs", LEGACY_CONFIG_AUDIT_LOG_FILENAME); } export function formatConfigOverwriteLogMessage(params: { @@ -307,13 +411,12 @@ export function finalizeConfigWriteAuditRecord(params: { nextNlink: success ? nextMetadata.nlink : null, nextUid: success ? nextMetadata.uid : null, nextGid: success ? nextMetadata.gid : null, - errorCode, - errorMessage, + ...(errorCode !== undefined ? { errorCode } : {}), + ...(errorMessage !== undefined ? { errorMessage } : {}), }; } type ConfigAuditAppendContext = { - fs: ConfigAuditFs; env: NodeJS.ProcessEnv; homedir: () => string; }; @@ -328,10 +431,10 @@ type ConfigAuditAppendParams = ConfigAuditAppendContext & function resolveConfigAuditAppendRecord(params: ConfigAuditAppendParams): ConfigAuditRecord { if ("record" in params) { - return redactSecrets(params.record); + return params.record; } - const { fs: _fs, env: _env, homedir: _homedir, ...record } = params; - return redactSecrets(record as ConfigAuditRecord); + const { env: _env, homedir: _homedir, ...record } = params; + return record as ConfigAuditRecord; } export type ConfigAuditScrubResult = { @@ -376,7 +479,7 @@ export async function scrubConfigAuditLog(params: { homedir: () => string; dryRun?: boolean; }): Promise { - const auditPath = resolveConfigAuditLogPath(params.env, params.homedir); + const auditPath = resolveLegacyConfigAuditLogPath(params.env, params.homedir); let raw: string; try { raw = await params.fs.promises.readFile(auditPath, "utf-8"); @@ -503,15 +606,43 @@ export async function scrubConfigAuditLog(params: { return { scanned, rewritten, skipped, aborted: false }; } +function resolveConfigAuditStoreEnv(params: { + env: NodeJS.ProcessEnv; + homedir: () => string; +}): NodeJS.ProcessEnv { + return { + ...params.env, + OPENCLAW_STATE_DIR: resolveStateDir(params.env, params.homedir), + }; +} + +function openConfigAuditStore(env: NodeJS.ProcessEnv) { + return createSqliteAuditRecordStore({ + scope: CONFIG_AUDIT_SCOPE, + maxEntries: CONFIG_AUDIT_MAX_ENTRIES, + env, + }); +} + +function configAuditEntryKey(record: ConfigAuditRecord): string { + return `${record.ts}:${record.event}:${randomUUID()}`; +} + +export function sanitizeConfigAuditRecord(record: ConfigAuditRecord): ConfigAuditRecord { + const sanitized = structuredClone(record); + sanitized.argv = redactConfigAuditArgv(capArgv(sanitized.argv)); + sanitized.execArgv = redactConfigAuditArgv(capArgv(sanitized.execArgv)); + return redactSecrets(sanitized); +} + export async function appendConfigAuditRecord(params: ConfigAuditAppendParams): Promise { try { - const auditPath = resolveConfigAuditLogPath(params.env, params.homedir); - const record = resolveConfigAuditAppendRecord(params); - await params.fs.promises.mkdir(path.dirname(auditPath), { recursive: true, mode: 0o700 }); - await params.fs.promises.appendFile(auditPath, `${JSON.stringify(record)}\n`, { - encoding: "utf-8", - mode: 0o600, - }); + const record = sanitizeConfigAuditRecord(resolveConfigAuditAppendRecord(params)); + openConfigAuditStore(resolveConfigAuditStoreEnv(params)).register( + configAuditEntryKey(record), + record, + Date.parse(record.ts), + ); } catch { // best-effort } @@ -519,13 +650,12 @@ export async function appendConfigAuditRecord(params: ConfigAuditAppendParams): export function appendConfigAuditRecordSync(params: ConfigAuditAppendParams): void { try { - const auditPath = resolveConfigAuditLogPath(params.env, params.homedir); - const record = resolveConfigAuditAppendRecord(params); - params.fs.mkdirSync(path.dirname(auditPath), { recursive: true, mode: 0o700 }); - params.fs.appendFileSync(auditPath, `${JSON.stringify(record)}\n`, { - encoding: "utf-8", - mode: 0o600, - }); + const record = sanitizeConfigAuditRecord(resolveConfigAuditAppendRecord(params)); + openConfigAuditStore(resolveConfigAuditStoreEnv(params)).register( + configAuditEntryKey(record), + record, + Date.parse(record.ts), + ); } catch { // best-effort } diff --git a/src/config/io.observe-recovery.test.ts b/src/config/io.observe-recovery.test.ts index b9a124dea85..b5c5b110301 100644 --- a/src/config/io.observe-recovery.test.ts +++ b/src/config/io.observe-recovery.test.ts @@ -11,6 +11,7 @@ import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, } from "../state/openclaw-state-db.js"; +import { listConfigAuditRecordsForTests } from "./io.audit.test-support.js"; import { createConfigIO } from "./io.js"; import { maybeRecoverSuspiciousConfigRead, @@ -101,17 +102,11 @@ describe("config observe recovery", () => { } async function readObserveEvents(auditPath: string): Promise[]> { - const events: Record[] = []; - for (const line of (await fsp.readFile(auditPath, "utf-8")).trim().split("\n")) { - if (!line) { - continue; - } - const parsed = JSON.parse(line) as Record; - if (parsed.event === "config.observe") { - events.push(parsed); - } - } - return events; + const stateDir = path.dirname(path.dirname(auditPath)); + return listConfigAuditRecordsForTests({ + env: { OPENCLAW_STATE_DIR: stateDir }, + homedir: () => stateDir, + }).filter((event) => event.event === "config.observe"); } async function listClobberFiles(configPath: string): Promise { diff --git a/src/config/io.observe-recovery.ts b/src/config/io.observe-recovery.ts index fae343b3b2a..3d68a315df5 100644 --- a/src/config/io.observe-recovery.ts +++ b/src/config/io.observe-recovery.ts @@ -240,7 +240,6 @@ function createConfigObserveAuditAppendParams( params: ConfigObserveAuditRecordParams, ) { return { - fs: deps.fs, env: deps.env, homedir: deps.homedir, record: createConfigObserveAuditRecord(params), diff --git a/src/config/io.observe.ts b/src/config/io.observe.ts index a3bde2b2ff5..8d2550d1b4f 100644 --- a/src/config/io.observe.ts +++ b/src/config/io.observe.ts @@ -259,7 +259,6 @@ export async function observeConfigSnapshot( (baseline?.hash ? baseline : null) ?? (await readConfigFingerprintForPath(deps, backupPath)); deps.logger.warn(`Config observe anomaly: ${snapshot.path} (${suspicious.join(", ")})`); await appendConfigAuditRecord({ - fs: deps.fs, env: deps.env, homedir: deps.homedir, record: createObserveAuditRecord({ snapshot, current, suspicious, entry, backup }), @@ -306,7 +305,6 @@ export function observeConfigSnapshotSync( (baseline?.hash ? baseline : null) ?? readConfigFingerprintForPathSync(deps, backupPath); deps.logger.warn(`Config observe anomaly: ${snapshot.path} (${suspicious.join(", ")})`); appendConfigAuditRecordSync({ - fs: deps.fs, env: deps.env, homedir: deps.homedir, record: createObserveAuditRecord({ snapshot, current, suspicious, entry, backup }), diff --git a/src/config/io.write.ts b/src/config/io.write.ts index 20957195264..73fc58a345a 100644 --- a/src/config/io.write.ts +++ b/src/config/io.write.ts @@ -294,7 +294,6 @@ export async function writeConfigFileFromContext( nextStat?: fs.Stats | null, ) => { await appendConfigAuditRecord({ - fs: deps.fs, env: deps.env, homedir: deps.homedir, record: finalizeConfigWriteAuditRecord({ diff --git a/src/infra/backup-create.test.ts b/src/infra/backup-create.test.ts index 48322101a75..cf9d176960c 100644 --- a/src/infra/backup-create.test.ts +++ b/src/infra/backup-create.test.ts @@ -8,6 +8,7 @@ import * as tar from "tar"; import { describe, expect, it, vi } from "vitest"; import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; import { backupVerifyCommand } from "../commands/backup-verify.js"; +import { CONFIG_AUDIT_MAX_ENTRIES, CONFIG_AUDIT_SCOPE } from "../config/io.audit.js"; import type { RuntimeEnv } from "../runtime.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { @@ -29,6 +30,8 @@ import { writeTarArchiveWithRetry } from "./backup-tar-retry.js"; import { isVolatileBackupPath } from "./backup-volatile-filter.js"; import { createBackupVolatileStatCache } from "./backup-volatile-stat-cache.js"; import { requireNodeSqlite } from "./node-sqlite.js"; +import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js"; +import { detectLegacyAuditLogs, migrateLegacyAuditLogs } from "./state-migrations.audit-logs.js"; function makeResult(overrides: Partial = {}): BackupCreateResult { return { @@ -239,6 +242,27 @@ describe("sanitizeOpenClawGlobalStateSnapshot", () => { database.close(); } }); + + it("leaves diagnostic state to its backup-specific sanitizer", () => { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(":memory:"); + try { + database.exec(` + CREATE TABLE diagnostic_events (scope TEXT); + INSERT INTO diagnostic_events VALUES ('migration.legacy-audit-raw'); + INSERT INTO diagnostic_events VALUES ('system-agent.audit'); + `); + + sanitizeOpenClawGlobalStateSnapshot(database); + + expect(database.prepare("SELECT scope FROM diagnostic_events").all()).toEqual([ + { scope: "migration.legacy-audit-raw" }, + { scope: "system-agent.audit" }, + ]); + } finally { + database.close(); + } + }); }); describe("writeTarArchiveWithRetry", () => { @@ -621,6 +645,235 @@ describe("createBackupArchive", () => { ); }); + it("replaces legacy audit raw archives with sanitized restorable snapshots", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-backup-audit-raw-", + scenario: "minimal", + }, + async (state) => { + const outputDir = state.path("backups"); + const extractDir = state.path("extract"); + const rawRelativePath = "logs/config-audit.jsonl.migrated.raw"; + const marker = "audit-value-7f3c"; + await fs.mkdir(outputDir, { recursive: true }); + await fs.mkdir(extractDir, { recursive: true }); + await state.writeText( + rawRelativePath, + `${JSON.stringify({ + ts: "2026-07-01T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "token", marker], + execArgv: [], + })}\n`, + ); + const { db } = openOpenClawStateDatabase({ env: state.env }); + db.prepare( + ` + INSERT INTO diagnostic_events ( + scope, event_key, payload_json, created_at, sequence + ) VALUES ('migration.legacy-audit-raw', 'checkpoint', '{}', 1, 1) + `, + ).run(); + + try { + const result = await createBackupArchive({ + output: outputDir, + includeWorkspace: false, + nowMs: Date.UTC(2026, 4, 9, 8, 15, 0), + }); + const entries = await listArchiveEntries(result.archivePath); + const rawEntry = expectDefined( + entries.find((entry) => entry.endsWith(`/state/${rawRelativePath}`)), + "sanitized raw archive entry", + ); + const databaseEntry = expectDefined( + entries.find((entry) => entry.endsWith("/state/state/openclaw.sqlite")), + "global state database entry", + ); + expect(entries.some((entry) => entry.endsWith(".doctor-scrub-restore"))).toBe(false); + + await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir }); + const archivedRaw = await fs.readFile(path.join(extractDir, rawEntry), "utf8"); + expect(archivedRaw).not.toContain(marker); + expect(JSON.parse(archivedRaw.trim())).toMatchObject({ + argv: ["openclaw", "config", "set", "token", "***"], + }); + const sqlite = requireNodeSqlite(); + const archivedDb = new sqlite.DatabaseSync(path.join(extractDir, databaseEntry), { + readOnly: true, + }); + try { + expect( + archivedDb + .prepare( + "SELECT COUNT(*) AS count FROM diagnostic_events WHERE scope = 'migration.legacy-audit-raw'", + ) + .get(), + ).toEqual({ count: 0 }); + } finally { + archivedDb.close(); + } + } finally { + closeOpenClawStateDatabase(); + } + }, + ); + }); + + it("omits completed blank audit append pads when dropping their checkpoints", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-backup-completed-audit-pad-", + scenario: "minimal", + }, + async (state) => { + const outputDir = state.path("backups"); + const extractDir = state.path("extract"); + const sourcePath = state.statePath("logs/config-audit.jsonl"); + const rawRelativePath = "logs/config-audit.jsonl.migrated.raw"; + await fs.mkdir(outputDir, { recursive: true }); + await fs.mkdir(extractDir, { recursive: true }); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile( + sourcePath, + `${JSON.stringify({ + ts: "2026-07-01T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "safe", "value"], + execArgv: [], + })}\n`, + ); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ + stateDir: state.stateDir, + doctorOnlyStateMigrations: true, + }), + stateDir: state.stateDir, + }); + expect( + detectLegacyAuditLogs({ + stateDir: state.stateDir, + doctorOnlyStateMigrations: true, + }).hasLegacy, + ).toBe(false); + const { db } = openOpenClawStateDatabase({ env: state.env }); + expect( + db + .prepare( + "SELECT COUNT(*) AS count FROM diagnostic_events WHERE scope = 'migration.legacy-audit-raw'", + ) + .get(), + ).toEqual({ count: 1 }); + + try { + const result = await createBackupArchive({ + output: outputDir, + includeWorkspace: false, + nowMs: Date.UTC(2026, 4, 9, 8, 20, 0), + }); + const entries = await listArchiveEntries(result.archivePath); + expect(entries.some((entry) => entry.endsWith(`/state/${rawRelativePath}`))).toBe(false); + const databaseEntry = expectDefined( + entries.find((entry) => entry.endsWith("/state/state/openclaw.sqlite")), + "global state database entry", + ); + await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir }); + const sqlite = requireNodeSqlite(); + const archivedDb = new sqlite.DatabaseSync(path.join(extractDir, databaseEntry), { + readOnly: true, + }); + try { + expect( + archivedDb + .prepare( + "SELECT COUNT(*) AS count FROM diagnostic_events WHERE scope = 'migration.legacy-audit-raw'", + ) + .get(), + ).toEqual({ count: 0 }); + } finally { + archivedDb.close(); + } + } finally { + closeOpenClawStateDatabase(); + } + }, + ); + }); + + it("preserves audit ordinals for identical later appends across backup restore", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-backup-audit-ordinal-", + scenario: "minimal", + }, + async (state) => { + const outputDir = state.path("backups"); + const extractDir = state.path("extract"); + const sourcePath = state.statePath("logs/config-audit.jsonl"); + const rawRelativePath = "logs/config-audit.jsonl.migrated.raw"; + const record = { + ts: "2026-07-01T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "safe", "same"], + execArgv: [], + }; + await fs.mkdir(outputDir, { recursive: true }); + await fs.mkdir(extractDir, { recursive: true }); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, `${JSON.stringify(record)}\n`); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ + stateDir: state.stateDir, + doctorOnlyStateMigrations: true, + }), + stateDir: state.stateDir, + }); + await fs.appendFile(state.statePath(rawRelativePath), `${JSON.stringify(record)}\n`); + + const result = await createBackupArchive({ + output: outputDir, + includeWorkspace: false, + nowMs: Date.UTC(2026, 4, 9, 8, 25, 0), + }); + const entries = await listArchiveEntries(result.archivePath); + const databaseEntry = expectDefined( + entries.find((entry) => entry.endsWith("/state/state/openclaw.sqlite")), + "global state database entry", + ); + await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir }); + closeOpenClawStateDatabase(); + + const restoredDatabasePath = path.join(extractDir, databaseEntry); + const restoredStateDir = path.dirname(path.dirname(restoredDatabasePath)); + const restoredDetection = detectLegacyAuditLogs({ + stateDir: restoredStateDir, + doctorOnlyStateMigrations: true, + }); + expect(restoredDetection.hasLegacy).toBe(true); + await migrateLegacyAuditLogs({ + detected: restoredDetection, + stateDir: restoredStateDir, + }); + const restoredEntries = createSqliteAuditRecordStore({ + scope: CONFIG_AUDIT_SCOPE, + maxEntries: CONFIG_AUDIT_MAX_ENTRIES, + env: { ...process.env, OPENCLAW_STATE_DIR: restoredStateDir }, + }).entries(); + expect(restoredEntries).toHaveLength(2); + expect(new Set(restoredEntries.map((entry) => entry.key)).size).toBe(2); + expect(restoredEntries.map((entry) => entry.value)).toEqual([record, record]); + closeOpenClawStateDatabase(); + }, + ); + }); + it("scrubs transient SQLite queue and plugin blob rows from archive snapshots", async () => { await withOpenClawTestState( { diff --git a/src/infra/backup-create.ts b/src/infra/backup-create.ts index 7e87dd4f540..457f2c2a69d 100644 --- a/src/infra/backup-create.ts +++ b/src/infra/backup-create.ts @@ -33,6 +33,14 @@ import { formatErrorMessage } from "./errors.js"; import { sameFileIdentity } from "./fs-safe-advanced.js"; import { writeJson } from "./json-files.js"; import { createVerifiedSqliteSnapshot } from "./sqlite-snapshot.js"; +import { + createLegacyAuditBackupSnapshots, + hasLegacyAuditBackupSources, + isLegacyAuditMigrationBackupPath, + rewriteLegacyAuditBackupCheckpoints, + type LegacyAuditBackupSnapshot, +} from "./state-migrations.audit-backup.js"; +import { withLegacyAuditMigrationLease } from "./state-migrations.audit-coordination.js"; const loadTarRuntime = createLazyRuntimeModule(() => import("tar")); @@ -618,6 +626,7 @@ async function createStateSqliteBackupPlan(params: { stateDir: string; tempDir: string; preservedStatePaths?: readonly string[]; + legacyAuditSnapshots: readonly LegacyAuditBackupSnapshot[]; }): Promise { // Complete discovery before writing snapshots. chooseBackupTempRoot keeps // tempDir outside stateDir, and this ordering prevents future overlap from @@ -676,7 +685,10 @@ async function createStateSqliteBackupPlan(params: { // Agent coordination is transient, while unrelated plugin databases // remain owner-defined. Queue and TTL-blob policy is global-only. transform: isGlobalStateDatabase - ? sanitizeOpenClawGlobalStateSnapshot + ? (database) => { + sanitizeOpenClawGlobalStateSnapshot(database); + rewriteLegacyAuditBackupCheckpoints(database, params.legacyAuditSnapshots); + } : canonicalAgentSource ? sanitizeOpenClawStateLeaseRows : undefined, @@ -771,19 +783,47 @@ export async function createBackupArchive( .map((asset) => asset.sourcePath), ].filter((entry) => stateAsset && isPathWithin(entry, stateAsset.sourcePath)); try { - const stateSqliteBackup = stateAsset - ? await createStateSqliteBackupPlan({ - stateDir: stateAsset.sourcePath, - tempDir, - preservedStatePaths, - }) - : { snapshots: [], discoveredSourcePaths: new Set() }; + // Capture every legacy file first, including active and claimed sources. + // A concurrent Doctor then leaves each row in this snapshot, the later + // SQLite snapshot, or both; restore-side import keys make overlap harmless. + const hasLegacyAuditSources = stateAsset + ? await hasLegacyAuditBackupSources(stateAsset.sourcePath) + : false; + const createSnapshotPlans = async () => { + const legacyAuditSnapshots = + stateAsset && hasLegacyAuditSources + ? await createLegacyAuditBackupSnapshots({ + stateDir: stateAsset.sourcePath, + tempDir, + }) + : []; + const stateSqliteBackup = stateAsset + ? await createStateSqliteBackupPlan({ + stateDir: stateAsset.sourcePath, + tempDir, + preservedStatePaths, + legacyAuditSnapshots, + }) + : { snapshots: [], discoveredSourcePaths: new Set() }; + return { legacyAuditSnapshots, stateSqliteBackup }; + }; + const snapshotPlans = + stateAsset && hasLegacyAuditSources + ? await withLegacyAuditMigrationLease(stateAsset.sourcePath, createSnapshotPlans) + : await createSnapshotPlans(); + const { legacyAuditSnapshots, stateSqliteBackup } = snapshotPlans; const sourcePathRemaps = new Map(); - const skippedSqliteSourcePaths = new Set(); + const skippedStateSourcePaths = new Set(); for (const snapshot of stateSqliteBackup.snapshots) { sourcePathRemaps.set(path.resolve(snapshot.sourcePath), snapshot.archiveSourcePath); for (const skippedSourcePath of snapshot.skippedSourcePaths) { - skippedSqliteSourcePaths.add(skippedSourcePath); + skippedStateSourcePaths.add(skippedSourcePath); + } + } + for (const snapshot of legacyAuditSnapshots) { + sourcePathRemaps.set(path.resolve(snapshot.sourcePath), snapshot.archiveSourcePath); + for (const skippedSourcePath of snapshot.skippedSourcePaths) { + skippedStateSourcePaths.add(skippedSourcePath); } } const manifest = buildManifest({ @@ -822,13 +862,19 @@ export async function createBackupArchive( if (stateFilter && !stateFilter(entryPath)) { return false; } + if ( + stateAsset && + isLegacyAuditMigrationBackupPath(resolvedEntryPath, stateAsset.sourcePath) + ) { + return false; + } const sqliteSourceKind = stateAsset ? classifyStateSqliteBackupSourcePath(resolvedEntryPath, stateAsset.sourcePath) : undefined; if (sqliteSourceKind === "excluded") { return false; } - if (skippedSqliteSourcePaths.has(resolvedEntryPath)) { + if (skippedStateSourcePaths.has(resolvedEntryPath)) { return false; } if ( @@ -878,6 +924,7 @@ export async function createBackupArchive( [ manifestPath, ...stateSqliteBackup.snapshots.map((snapshot) => snapshot.sourcePath), + ...legacyAuditSnapshots.map((snapshot) => snapshot.sourcePath), ...result.assets.map((asset) => asset.sourcePath), ], ), diff --git a/src/infra/backup-volatile-filter.test.ts b/src/infra/backup-volatile-filter.test.ts index 847117a575a..25abfd7fa78 100644 --- a/src/infra/backup-volatile-filter.test.ts +++ b/src/infra/backup-volatile-filter.test.ts @@ -22,6 +22,12 @@ describe("isVolatileBackupPath", () => { [`${stateDir}/ipc/gateway.sock`, true], [`${stateDir}/gateway.pid`, true], [`${stateDir}/tmp/pending.tmp`, true], + [`${stateDir}/audit/system-agent.jsonl.migrated.raw`, false], + [`${stateDir}/audit/system-agent.jsonl.migrated.2.raw`, false], + [`${stateDir}/audit/system-agent.jsonl.migrated.10.raw`, false], + [`${stateDir}/audit/system-agent.jsonl.migrated.raw.doctor-scrub-restore`, false], + [`${stateDir}/logs/config-audit.jsonl.migrated.raw.doctor-scrub-staging`, false], + [`${stateDir}/logs/config-audit.jsonl.migrated.raw.doctor-scrub-progress`, false], [`${stateDir}/delivery-queue/pending.tmp`, true], [`${stateDir}/session-delivery-queue/pending.tmp`, true], @@ -38,6 +44,7 @@ describe("isVolatileBackupPath", () => { ["/home/user/project/README.md", false], ["/home/user/project/Cargo.lock", false], ["/home/user/project/pending.tmp", false], + ["/home/user/project/config.jsonl.doctor-scrub-restore", false], // non-volatile: log-like name outside scope ["/home/user/notes/daily.log", false], ])("classifies %s as volatile=%s", (p, expected) => { diff --git a/src/infra/fs-safe-advanced.ts b/src/infra/fs-safe-advanced.ts index 695fe7ea915..c90f1a434a5 100644 --- a/src/infra/fs-safe-advanced.ts +++ b/src/infra/fs-safe-advanced.ts @@ -5,6 +5,7 @@ import "./fs-safe-defaults.js"; export { assertNoSymlinkParents, assertNoSymlinkParentsSync, + type FileIdentityStat, sameFileIdentity, sanitizeUntrustedFileName, } from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/regular-file.ts b/src/infra/regular-file.ts index faf6e036661..d503a1bf415 100644 --- a/src/infra/regular-file.ts +++ b/src/infra/regular-file.ts @@ -4,7 +4,6 @@ import "./fs-safe-defaults.js"; // Regular-file IO helpers reject symlinks and non-file targets before reads or // appends touch user-controlled paths. export { - appendRegularFile, appendRegularFileSync, readRegularFile, readRegularFileSync, diff --git a/src/infra/sqlite-audit-record-store.test.ts b/src/infra/sqlite-audit-record-store.test.ts new file mode 100644 index 00000000000..b4037322870 --- /dev/null +++ b/src/infra/sqlite-audit-record-store.test.ts @@ -0,0 +1,120 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { closeOpenClawStateDatabase } from "../state/openclaw-state-db.js"; +import { withTempDir } from "../test-helpers/temp-dir.js"; +import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js"; + +describe("SQLite audit record store", () => { + afterEach(() => { + closeOpenClawStateDatabase(); + }); + + it("keeps the newest configured number of rows per scope", async () => { + await withTempDir({ prefix: "openclaw-audit-store-" }, async (stateDir) => { + const store = createSqliteAuditRecordStore<{ value: number }>({ + scope: "bounded-test", + maxEntries: 2, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + + store.register("one", { value: 1 }, 1); + store.register("two", { value: 2 }, 2); + store.register("three", { value: 3 }, 3); + + expect(store.size()).toBe(2); + expect(store.entries().map((entry) => entry.key)).toEqual(["two", "three"]); + }); + }); + + it("preserves insertion order and prunes the oldest row when timestamps tie", async () => { + await withTempDir({ prefix: "openclaw-audit-store-ties-" }, async (stateDir) => { + const store = createSqliteAuditRecordStore<{ value: number }>({ + scope: "tied-timestamps", + maxEntries: 2, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + + store.register("z-first", { value: 1 }, 1); + store.register("a-second", { value: 2 }, 1); + expect(store.entries().map((entry) => entry.key)).toEqual(["z-first", "a-second"]); + + store.register("m-third", { value: 3 }, 1); + expect(store.entries().map((entry) => entry.key)).toEqual(["a-second", "m-third"]); + }); + }); + + it("prunes by insertion order when wall-clock timestamps move", async () => { + await withTempDir({ prefix: "openclaw-audit-store-clock-skew-" }, async (stateDir) => { + const store = createSqliteAuditRecordStore<{ value: number }>({ + scope: "clock-skew", + maxEntries: 2, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + + store.register("future-first", { value: 1 }, 4_000_000_000_000); + store.register("past-second", { value: 2 }, 1); + store.register("current-third", { value: 3 }, 2_000_000_000_000); + + expect(store.entries().map((entry) => entry.key)).toEqual(["past-second", "current-third"]); + }); + }); + + it("commits batch inserts with one retention pass", async () => { + await withTempDir({ prefix: "openclaw-audit-store-batch-" }, async (stateDir) => { + const store = createSqliteAuditRecordStore<{ value: number }>({ + scope: "batch-test", + maxEntries: 2, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + + store.registerLegacyMany([ + { key: "one", value: { value: 1 }, createdAt: 1 }, + { key: "two", value: { value: 2 }, createdAt: 2 }, + { key: "three", value: { value: 3 }, createdAt: 3 }, + ]); + + expect(store.entries().map((entry) => entry.key)).toEqual(["two", "three"]); + }); + }); + + it("updates a retained key without consuming another bounded entry", async () => { + await withTempDir({ prefix: "openclaw-audit-store-upsert-" }, async (stateDir) => { + const store = createSqliteAuditRecordStore<{ value: number }>({ + scope: "upsert-test", + maxEntries: 2, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + + store.register("one", { value: 1 }, 1); + store.register("two", { value: 2 }, 2); + store.upsert("one", { value: 3 }, 3); + + expect(store.size()).toBe(2); + expect(store.entries()).toEqual([ + { key: "one", value: { value: 3 }, createdAt: 3 }, + { key: "two", value: { value: 2 }, createdAt: 2 }, + ]); + }); + }); + + it("orders legacy batches before existing runtime rows", async () => { + await withTempDir({ prefix: "openclaw-audit-store-legacy-order-" }, async (stateDir) => { + const store = createSqliteAuditRecordStore<{ value: number }>({ + scope: "legacy-order", + maxEntries: 4, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + + store.register("runtime", { value: 3 }, 3); + store.registerLegacyMany([ + { key: "legacy-one", value: { value: 1 }, createdAt: 1 }, + { key: "legacy-two", value: { value: 2 }, createdAt: 2 }, + ]); + + expect(store.entries().map((entry) => entry.key)).toEqual([ + "legacy-one", + "legacy-two", + "runtime", + ]); + }); + }); +}); diff --git a/src/infra/sqlite-audit-record-store.ts b/src/infra/sqlite-audit-record-store.ts new file mode 100644 index 00000000000..2461a00836e --- /dev/null +++ b/src/infra/sqlite-audit-record-store.ts @@ -0,0 +1,222 @@ +// Shared SQLite storage for append-only diagnostic audit records. +import type { DatabaseSync } from "node:sqlite"; +import type { Selectable } from "kysely"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; + +type DiagnosticEventsTable = OpenClawStateKyselyDatabase["diagnostic_events"]; +type AuditRecordDatabase = Pick; +type DiagnosticEventRow = Pick< + Selectable, + "event_key" | "payload_json" | "created_at" | "sequence" +>; +type PreparedDiagnosticEventRow = Omit; + +const LEGACY_AUDIT_SEQUENCE_BASE = Number.MIN_SAFE_INTEGER; + +type SqliteAuditRecordEntry = { + key: string; + value: T; + createdAt: number; +}; + +function getAuditRecordKysely(database: DatabaseSync) { + return getNodeSqliteKysely(database); +} + +function parseAuditRecord(row: DiagnosticEventRow): SqliteAuditRecordEntry { + return { + key: row.event_key, + value: JSON.parse(row.payload_json) as T, + createdAt: row.created_at, + }; +} + +function countAuditRecords(database: DatabaseSync, scope: string): number { + const row = executeSqliteQueryTakeFirstSync( + database, + getAuditRecordKysely(database) + .selectFrom("diagnostic_events") + .select((eb) => eb.fn.countAll().as("count")) + .where("scope", "=", scope), + ); + return typeof row?.count === "bigint" ? Number(row.count) : (row?.count ?? 0); +} + +function nextAuditSequence(params: { + database: DatabaseSync; + scope: string; + legacy: boolean; +}): number { + const row = executeSqliteQueryTakeFirstSync( + params.database, + getAuditRecordKysely(params.database) + .selectFrom("diagnostic_events") + .select((eb) => eb.fn.max("sequence").as("sequence")) + .where("scope", "=", params.scope) + .where("sequence", params.legacy ? "<" : ">=", 0), + ); + const current = row?.sequence ?? (params.legacy ? LEGACY_AUDIT_SEQUENCE_BASE : 0); + const next = current + 1; + if (!Number.isSafeInteger(next) || (params.legacy && next >= 0)) { + throw new Error(`Audit sequence exhausted for scope ${params.scope}`); + } + return next; +} + +function pruneAuditRecords(params: { + database: DatabaseSync; + scope: string; + maxEntries: number; + protectedKey?: string; +}): void { + const overflow = countAuditRecords(params.database, params.scope) - params.maxEntries; + if (overflow <= 0) { + return; + } + const protectedKey = params.protectedKey; + const baseCandidates = getAuditRecordKysely(params.database) + .selectFrom("diagnostic_events") + .select("event_key") + .where("scope", "=", params.scope); + const candidates = ( + protectedKey === undefined + ? baseCandidates + : baseCandidates.where("event_key", "!=", protectedKey) + ) + .orderBy("sequence", "asc") + .limit(overflow); + const rows = executeSqliteQuerySync(params.database, candidates).rows; + for (const row of rows) { + executeSqliteQuerySync( + params.database, + getAuditRecordKysely(params.database) + .deleteFrom("diagnostic_events") + .where("scope", "=", params.scope) + .where("event_key", "=", row.event_key), + ); + } +} + +/** Opens one bounded append-only audit scope in the shared state database. */ +export function createSqliteAuditRecordStore( + options: OpenClawStateDatabaseOptions & { scope: string; maxEntries: number }, +) { + const scope = options.scope; + const maxEntries = Math.max(1, Math.floor(options.maxEntries)); + function prepareRecord(record: SqliteAuditRecordEntry): PreparedDiagnosticEventRow { + const payloadJson = JSON.stringify(record.value); + if (payloadJson === undefined) { + throw new Error(`Audit record ${scope}/${record.key} is not JSON-serializable`); + } + return { + event_key: record.key, + payload_json: payloadJson, + created_at: record.createdAt, + }; + } + + function insertRecord(database: DatabaseSync, record: DiagnosticEventRow): void { + executeSqliteQuerySync( + database, + getAuditRecordKysely(database) + .insertInto("diagnostic_events") + .values({ + scope, + event_key: record.event_key, + payload_json: record.payload_json, + created_at: record.created_at, + sequence: record.sequence, + }) + .onConflict((conflict) => conflict.columns(["scope", "event_key"]).doNothing()), + ); + } + + return { + register(key: string, value: T, createdAt = Date.now()): void { + const record = prepareRecord({ key, value, createdAt }); + runOpenClawStateWriteTransaction((database) => { + insertRecord(database.db, { + ...record, + sequence: nextAuditSequence({ database: database.db, scope, legacy: false }), + }); + // Audit retention is scope-local. Keep the just-written row and evict the oldest + // prior rows in the same synchronous commit section. + pruneAuditRecords({ + database: database.db, + scope, + maxEntries, + protectedKey: key, + }); + }, options); + }, + upsert(key: string, value: T, createdAt = Date.now()): void { + const record = prepareRecord({ key, value, createdAt }); + runOpenClawStateWriteTransaction((database) => { + executeSqliteQuerySync( + database.db, + getAuditRecordKysely(database.db) + .insertInto("diagnostic_events") + .values({ + scope, + event_key: record.event_key, + payload_json: record.payload_json, + created_at: record.created_at, + sequence: nextAuditSequence({ database: database.db, scope, legacy: false }), + }) + .onConflict((conflict) => + conflict.columns(["scope", "event_key"]).doUpdateSet({ + payload_json: record.payload_json, + created_at: record.created_at, + }), + ), + ); + pruneAuditRecords({ + database: database.db, + scope, + maxEntries, + protectedKey: key, + }); + }, options); + }, + registerLegacyMany(records: readonly SqliteAuditRecordEntry[]): void { + const prepared = records.map(prepareRecord); + if (prepared.length === 0) { + return; + } + // Legacy imports can contain tens of thousands of rows. Serialize first, + // then assign ordered negative sequences before runtime audit history. + runOpenClawStateWriteTransaction((database) => { + let sequence = nextAuditSequence({ database: database.db, scope, legacy: true }); + for (const record of prepared) { + insertRecord(database.db, { ...record, sequence }); + sequence += 1; + } + pruneAuditRecords({ database: database.db, scope, maxEntries }); + }, options); + }, + size(): number { + return countAuditRecords(openOpenClawStateDatabase(options).db, scope); + }, + entries(): SqliteAuditRecordEntry[] { + const database = openOpenClawStateDatabase(options); + return executeSqliteQuerySync( + database.db, + getAuditRecordKysely(database.db) + .selectFrom("diagnostic_events") + .select(["event_key", "payload_json", "created_at", "sequence"]) + .where("scope", "=", scope) + .orderBy("sequence", "asc"), + ).rows.map((row) => parseAuditRecord(row)); + }, + }; +} diff --git a/src/infra/state-migrations.audit-backup.test.ts b/src/infra/state-migrations.audit-backup.test.ts new file mode 100644 index 00000000000..5c632e2253b --- /dev/null +++ b/src/infra/state-migrations.audit-backup.test.ts @@ -0,0 +1,242 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { expectDefined } from "@openclaw/normalization-core"; +import { afterEach, describe, expect, it } from "vitest"; +import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; +import { withTempDir } from "../test-helpers/temp-dir.js"; +import { + createLegacyAuditBackupSnapshots, + hasLegacyAuditBackupSources, + isLegacyAuditMigrationBackupPath, +} from "./state-migrations.audit-backup.js"; + +const TEST_SCRUB_PATTERN = Buffer.from( + Array.from({ length: 32 }, (_, index) => (index % 2 === 0 ? 0x20 : 0x09)), +); + +function configAuditRecord(value: string) { + return { + ts: "2026-07-01T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "token", value], + execArgv: [], + }; +} + +async function buildTestAuditRestoreJournal( + rawPath: string, + sourceRaw: Buffer, + scrubbedBytes = 0, +): Promise { + const stat = await fs.stat(rawPath); + const journal = `${JSON.stringify({ + schemaVersion: 6, + rawBase64: sourceRaw.toString("base64"), + scrubPatternBase64: TEST_SCRUB_PATTERN.toString("base64"), + target: { dev: stat.dev, ino: stat.ino, size: sourceRaw.length }, + })}\n`; + await fs.writeFile( + `${rawPath}.doctor-scrub-progress`, + `${JSON.stringify({ + schemaVersion: 1, + journalHash: createHash("sha256").update(journal).digest("hex"), + direction: "scrubbing", + committedBytes: scrubbedBytes, + pendingEnd: scrubbedBytes, + extentBytes: sourceRaw.length, + })}\n`, + ); + return journal; +} + +describe("legacy audit raw backup snapshots", () => { + afterEach(() => { + resetPluginStateStoreForTests(); + }); + + it("recognizes only supported audit migration paths", () => { + const stateDir = "/opt/openclaw/state"; + expect( + isLegacyAuditMigrationBackupPath( + `${stateDir}/logs/config-audit.jsonl.migrated.10.raw.doctor-scrub-restore`, + stateDir, + ), + ).toBe(true); + expect( + isLegacyAuditMigrationBackupPath( + `${stateDir}/audit/.system-agent.jsonl.doctor-importing.2`, + stateDir, + ), + ).toBe(true); + expect( + isLegacyAuditMigrationBackupPath( + `${stateDir}/logs/config-audit.jsonl.migrated.raw.doctor-scrub-progress`, + stateDir, + ), + ).toBe(true); + expect( + isLegacyAuditMigrationBackupPath( + `${stateDir}/plugins/example/cache.jsonl.migrated.raw`, + stateDir, + ), + ).toBe(false); + }); + + it("propagates audit-directory inspection failures", async () => { + await withTempDir({ prefix: "openclaw-audit-backup-inspection-" }, async (stateDir) => { + await fs.writeFile(path.join(stateDir, "logs"), "not a directory"); + + await expect(hasLegacyAuditBackupSources(stateDir)).rejects.toMatchObject({ + code: "ENOTDIR", + }); + }); + }); + + it("captures an active legacy source before the later SQLite snapshot", async () => { + await withTempDir({ prefix: "openclaw-audit-backup-active-" }, async (rootDir) => { + const stateDir = path.join(rootDir, "state"); + const tempDir = path.join(rootDir, "backup-temp"); + const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl"); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.mkdir(tempDir); + await fs.writeFile(sourcePath, `${JSON.stringify(configAuditRecord("active-value-7f3c"))}\n`); + + const snapshots = await createLegacyAuditBackupSnapshots({ stateDir, tempDir }); + const snapshotAsset = expectDefined(snapshots[0], "snapshot"); + const snapshot = await fs.readFile(snapshotAsset.sourcePath, "utf8"); + + expect(snapshotAsset.archiveSourcePath).toBe(sourcePath); + expect(snapshot).not.toContain("active-value-7f3c"); + expect(JSON.parse(snapshot.trim())).toMatchObject({ + argv: ["openclaw", "config", "set", "token", "***"], + }); + }); + }); + + it("captures a stable active prefix while an old writer keeps appending", async () => { + await withTempDir({ prefix: "openclaw-audit-backup-appending-" }, async (rootDir) => { + const stateDir = path.join(rootDir, "state"); + const tempDir = path.join(rootDir, "backup-temp"); + const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl"); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.mkdir(tempDir); + await fs.writeFile( + sourcePath, + Buffer.concat([ + Buffer.from(`${JSON.stringify(configAuditRecord("initial-value-7f3c"))}\n`), + Buffer.alloc(4 * 1024 * 1024, 0x20), + ]), + ); + + const snapshotPromise = createLegacyAuditBackupSnapshots({ stateDir, tempDir }); + for (let index = 0; index < 8; index += 1) { + await fs.appendFile( + sourcePath, + `${JSON.stringify(configAuditRecord(`late-value-${index}-9a21`))}\n`, + ); + } + const snapshots = await snapshotPromise; + const snapshotAsset = expectDefined(snapshots[0], "snapshot"); + const snapshot = await fs.readFile(snapshotAsset.sourcePath, "utf8"); + + expect(snapshot).not.toContain("initial-value-7f3c"); + expect(snapshot).not.toContain("late-value-"); + expect( + snapshot + .trim() + .split("\n") + .map((line) => JSON.parse(line)), + ).not.toHaveLength(0); + }); + }); + + it("captures an unimported append without archiving its secret", async () => { + await withTempDir({ prefix: "openclaw-audit-backup-" }, async (rootDir) => { + const stateDir = path.join(rootDir, "state"); + const tempDir = path.join(rootDir, "backup-temp"); + const rawPath = path.join(stateDir, "logs", "config-audit.jsonl.migrated.raw"); + await fs.mkdir(path.dirname(rawPath), { recursive: true }); + await fs.mkdir(tempDir); + await fs.writeFile(rawPath, `${JSON.stringify(configAuditRecord("late-value-7f3c"))}\n`); + + const snapshots = await createLegacyAuditBackupSnapshots({ stateDir, tempDir }); + + expect(snapshots).toHaveLength(1); + const snapshotAsset = expectDefined(snapshots[0], "snapshot"); + expect(snapshotAsset.archiveSourcePath).toBe(rawPath); + const snapshot = await fs.readFile(snapshotAsset.sourcePath, "utf8"); + expect(snapshot).not.toContain("late-value-7f3c"); + expect(JSON.parse(snapshot.trim())).toMatchObject({ + argv: ["openclaw", "config", "set", "token", "***"], + }); + }); + }); + + it("reconstructs a scrub-in-progress source and sanitizes its later append", async () => { + await withTempDir({ prefix: "openclaw-audit-backup-recovery-" }, async (rootDir) => { + const stateDir = path.join(rootDir, "state"); + const tempDir = path.join(rootDir, "backup-temp"); + const rawPath = path.join(stateDir, "logs", "config-audit.jsonl.migrated.raw"); + const original = Buffer.from(`${JSON.stringify(configAuditRecord("original-value-7f3c"))}\n`); + const later = `${JSON.stringify(configAuditRecord("later-value-9a21"))}\n`; + const partial = Buffer.from(original); + for (let index = 0; index < Math.floor(partial.length / 2); index += 1) { + partial[index] = TEST_SCRUB_PATTERN[index % TEST_SCRUB_PATTERN.length]!; + } + await fs.mkdir(path.dirname(rawPath), { recursive: true }); + await fs.mkdir(tempDir); + await fs.writeFile(rawPath, Buffer.concat([partial, Buffer.from(later)])); + await fs.writeFile( + `${rawPath}.doctor-scrub-restore`, + await buildTestAuditRestoreJournal(rawPath, original, Math.floor(partial.length / 2)), + ); + + const snapshots = await createLegacyAuditBackupSnapshots({ stateDir, tempDir }); + const snapshotAsset = expectDefined(snapshots[0], "snapshot"); + const snapshot = await fs.readFile(snapshotAsset.sourcePath, "utf8"); + + expect(snapshot).not.toContain("original-value-7f3c"); + expect(snapshot).not.toContain("later-value-9a21"); + expect( + snapshot + .trim() + .split("\n") + .map((line) => JSON.parse(line)), + ).toMatchObject([ + { argv: ["openclaw", "config", "set", "token", "***"] }, + { argv: ["openclaw", "config", "set", "token", "***"] }, + ]); + }); + }); + + it("ignores a stale restore journal after the raw archive is replaced", async () => { + await withTempDir({ prefix: "openclaw-audit-backup-stale-journal-" }, async (rootDir) => { + const stateDir = path.join(rootDir, "state"); + const tempDir = path.join(rootDir, "backup-temp"); + const rawPath = path.join(stateDir, "logs", "config-audit.jsonl.migrated.raw"); + const original = Buffer.from(`${JSON.stringify(configAuditRecord("old-value-7f3c"))}\n`); + const replacement = { + ...configAuditRecord("replacement-value-9a21"), + event: "config.delete", + }; + await fs.mkdir(path.dirname(rawPath), { recursive: true }); + await fs.mkdir(tempDir); + await fs.writeFile(rawPath, `${JSON.stringify(replacement)}\n`); + await fs.writeFile( + `${rawPath}.doctor-scrub-restore`, + await buildTestAuditRestoreJournal(rawPath, original), + ); + + const snapshots = await createLegacyAuditBackupSnapshots({ stateDir, tempDir }); + const snapshotAsset = expectDefined(snapshots[0], "snapshot"); + const snapshot = JSON.parse(await fs.readFile(snapshotAsset.sourcePath, "utf8")); + + expect(snapshot).toMatchObject({ + event: "config.delete", + argv: ["openclaw", "config", "set", "token", "***"], + }); + }); + }); +}); diff --git a/src/infra/state-migrations.audit-backup.ts b/src/infra/state-migrations.audit-backup.ts new file mode 100644 index 00000000000..deadf5f7b2a --- /dev/null +++ b/src/infra/state-migrations.audit-backup.ts @@ -0,0 +1,228 @@ +// Builds secret-sanitized backup replacements for legacy audit append archives. +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import { root as createFsSafeRoot } from "./fs-safe.js"; +import { + detectLegacyAuditLogs, + legacyAuditRawCheckpointKey, + legacyAuditSourceGenerationKey, + type LegacyAuditRawCheckpoint, +} from "./state-migrations.audit-checkpoints.js"; +import { + prepareLegacyAuditRecords, + serializePreparedAuditRecords, +} from "./state-migrations.audit-records.js"; +import { + findPreviousLegacyAuditRawCheckpoint, + readLegacyAuditRecoverySourceForBackup, + readLegacyAuditSourcePrefixSnapshotForBackup, +} from "./state-migrations.audit-recovery.js"; + +const LEGACY_AUDIT_LOGICAL_PATHS = [ + { directory: "logs", basename: "config-audit.jsonl" }, + { directory: "audit", basename: "system-agent.jsonl" }, + { directory: "audit", basename: "crestodian.jsonl" }, +] as const; + +export async function hasLegacyAuditBackupSources(stateDir: string): Promise { + for (const logical of LEGACY_AUDIT_LOGICAL_PATHS) { + let entries: string[]; + try { + entries = await fs.readdir(path.join(stateDir, logical.directory)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + continue; + } + throw error; + } + const escaped = logical.basename.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const sourcePattern = new RegExp( + `^(?:${escaped}|\\.${escaped}\\.doctor-importing(?:\\.(?:[2-9]|[1-9][0-9]+))?|${escaped}\\.migrated(?:\\.(?:[2-9]|[1-9][0-9]+))?\\.raw(?:\\.doctor-scrub-(?:progress|restore|staging))?)$`, + "u", + ); + if (entries.some((entry) => sourcePattern.test(entry))) { + return true; + } + } + return false; +} + +export function isLegacyAuditMigrationBackupPath(sourcePath: string, stateDir: string): boolean { + const relativePath = path.relative(path.resolve(stateDir), path.resolve(sourcePath)); + if (!relativePath || relativePath.startsWith(`..${path.sep}`) || path.isAbsolute(relativePath)) { + return false; + } + const directory = path.dirname(relativePath); + const basename = path.basename(relativePath); + for (const logical of LEGACY_AUDIT_LOGICAL_PATHS) { + if (directory !== logical.directory) { + continue; + } + if (basename === logical.basename) { + return true; + } + const escaped = logical.basename.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const claimPattern = new RegExp( + `^\\.${escaped}\\.doctor-importing(?:\\.(?:[2-9]|[1-9][0-9]+))?$`, + "u", + ); + const rawPattern = new RegExp( + `^${escaped}\\.migrated(?:\\.(?:[2-9]|[1-9][0-9]+))?\\.raw(?:\\.doctor-scrub-(?:progress|restore|staging))?$`, + "u", + ); + if (claimPattern.test(basename) || rawPattern.test(basename)) { + return true; + } + } + return false; +} + +type LegacyAuditBackupCheckpoint = { + key: string; + value: LegacyAuditRawCheckpoint; +}; + +export type LegacyAuditBackupSnapshot = { + sourcePath: string; + archiveSourcePath: string; + skippedSourcePaths: Set; + checkpoint?: LegacyAuditBackupCheckpoint; +}; + +/** Replaces live raw checkpoints with metadata for the transformed backup files. */ +export function rewriteLegacyAuditBackupCheckpoints( + database: DatabaseSync, + snapshots: readonly LegacyAuditBackupSnapshot[], +): void { + const hasDiagnosticEvents = database // sqlite-allow-raw -- Offline snapshot maintenance boundary. + .prepare("SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("diagnostic_events") as { ok?: unknown } | undefined; + if (hasDiagnosticEvents?.ok !== 1) { + return; + } + const scope = "migration.legacy-audit-raw"; + database.prepare("DELETE FROM diagnostic_events WHERE scope = ?").run(scope); // sqlite-allow-raw -- Offline snapshot maintenance boundary. + const insert = database // sqlite-allow-raw -- Offline snapshot maintenance boundary. + .prepare( + `INSERT INTO diagnostic_events ( + scope, event_key, payload_json, created_at, sequence + ) VALUES (?, ?, ?, ?, ?)`, + ); + let sequence = 1; + for (const snapshot of snapshots) { + if (!snapshot.checkpoint) { + continue; + } + insert.run( + scope, + snapshot.checkpoint.key, + JSON.stringify(snapshot.checkpoint.value), + 0, + sequence, + ); + sequence += 1; + } +} + +async function createLegacyAuditBackupSnapshotsOnce(params: { + stateDir: string; + tempDir: string; +}): Promise { + const detected = detectLegacyAuditLogs({ + stateDir: params.stateDir, + doctorOnlyStateMigrations: true, + }); + if (detected.sources.length === 0) { + return []; + } + const root = await createFsSafeRoot(params.stateDir, { + hardlinks: "reject", + maxBytes: Number.MAX_SAFE_INTEGER, + mkdir: false, + mode: 0o600, + symlinks: "reject", + }); + const snapshots: LegacyAuditBackupSnapshot[] = []; + for (const [index, source] of detected.sources.entries()) { + const sourceRelativePath = path.relative(path.resolve(params.stateDir), source.sourcePath); + const snapshot = + source.storage === "raw-archive" + ? await readLegacyAuditRecoverySourceForBackup(root, sourceRelativePath) + : await readLegacyAuditSourcePrefixSnapshotForBackup(root, sourceRelativePath); + const sourceGeneration = legacyAuditSourceGenerationKey(sourceRelativePath); + const previousCheckpoint = + source.storage === "raw-archive" + ? findPreviousLegacyAuditRawCheckpoint(params.stateDir, sourceRelativePath) + : undefined; + const prepared = prepareLegacyAuditRecords( + source, + snapshot.raw, + sourceGeneration, + previousCheckpoint?.recordOrdinalBase ?? 0, + ); + if (!prepared.ok) { + throw new Error( + `Legacy ${source.label} append archive cannot be sanitized for backup: ${prepared.warnings.join("; ")}`, + ); + } + const sourcePath = path.join(params.tempDir, `legacy-audit-raw-${index}.jsonl`); + await fs.writeFile(sourcePath, prepared.sanitizedJsonl, { mode: 0o600 }); + let checkpoint: LegacyAuditBackupCheckpoint | undefined; + if (previousCheckpoint) { + if (previousCheckpoint.recordCount > prepared.records.length) { + throw new Error( + `Legacy ${source.label} append archive is shorter than its durable checkpoint`, + ); + } + // Backup rewrites raw bytes to sanitized JSONL. Preserve the source ordinal + // and rebase the checkpoint hash onto the equivalent transformed prefix. + const transformedPrefix = Buffer.from( + serializePreparedAuditRecords(prepared.records.slice(0, previousCheckpoint.recordCount)), + "utf8", + ); + const value: LegacyAuditRawCheckpoint = { + ...previousCheckpoint, + dev: 0, + ino: 0, + mtimeMs: 0, + size: transformedPrefix.length, + contentHash: createHash("sha256").update(transformedPrefix).digest("hex"), + }; + checkpoint = { key: legacyAuditRawCheckpointKey(value), value }; + } + snapshots.push({ + sourcePath, + archiveSourcePath: source.sourcePath, + ...(checkpoint ? { checkpoint } : {}), + skippedSourcePaths: new Set([ + path.resolve(source.sourcePath), + path.resolve(`${source.sourcePath}.doctor-scrub-progress`), + path.resolve(`${source.sourcePath}.doctor-scrub-restore`), + path.resolve(`${source.sourcePath}.doctor-scrub-staging`), + ]), + }); + } + return snapshots; +} + +export async function createLegacyAuditBackupSnapshots(params: { + stateDir: string; + tempDir: string; +}): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + return await createLegacyAuditBackupSnapshotsOnce(params); + } catch (error) { + lastError = error; + if (attempt < 2) { + await new Promise((resolve) => { + setTimeout(resolve, 25); + }); + } + } + } + throw lastError; +} diff --git a/src/infra/state-migrations.audit-checkpoints.ts b/src/infra/state-migrations.audit-checkpoints.ts new file mode 100644 index 00000000000..5efbed057d9 --- /dev/null +++ b/src/infra/state-migrations.audit-checkpoints.ts @@ -0,0 +1,273 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js"; +import type { + LegacyAuditLogSource, + LegacyAuditLogsDetection, +} from "./state-migrations.audit-logs.types.js"; + +export type LegacyAuditFileCheckpoint = { + dev: number; + ino: number; + mtimeMs: number; + size: number; +}; + +export type LegacyAuditRawCheckpoint = LegacyAuditFileCheckpoint & { + phase: "merge-intent" | "raw"; + generationKey: string; + recordCount: number; + recordOrdinalBase: number; + contentHash: string; + sanitizedContentHash: string; + sanitizedSize: number; +}; + +const LEGACY_AUDIT_RAW_CHECKPOINT_SCOPE = "migration.legacy-audit-raw"; +const LEGACY_AUDIT_RAW_CHECKPOINT_MAX_ENTRIES = 10_000; + +export function legacyAuditRawCheckpointKey(checkpoint: LegacyAuditRawCheckpoint): string { + return checkpoint.generationKey; +} + +export function legacyAuditSourceGenerationKey(rawArchiveRelativePath: string): string { + // The numbered raw archive path is the durable generation identity. Unlike + // device/inode metadata, it survives backup restore and cross-device moves. + return createHash("sha256") + .update(rawArchiveRelativePath.replace(/\\/gu, "/")) + .digest("hex") + .slice(0, 16); +} + +export function openLegacyAuditRawCheckpointStore(stateDir: string) { + return createSqliteAuditRecordStore({ + scope: LEGACY_AUDIT_RAW_CHECKPOINT_SCOPE, + maxEntries: LEGACY_AUDIT_RAW_CHECKPOINT_MAX_ENTRIES, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); +} + +export function hasLegacyAuditRawCheckpointCapacity( + stateDir: string, + rawArchiveRelativePath: string, +): boolean { + const generationKey = legacyAuditSourceGenerationKey(rawArchiveRelativePath); + const entries = openLegacyAuditRawCheckpointStore(stateDir).entries(); + return ( + entries.some((entry) => entry.value.generationKey === generationKey) || + entries.length < LEGACY_AUDIT_RAW_CHECKPOINT_MAX_ENTRIES + ); +} + +function statLegacyAuditRawCheckpoint(sourcePath: string): LegacyAuditFileCheckpoint | undefined { + try { + const stat = fs.lstatSync(sourcePath); + if (!stat.isFile() || stat.isSymbolicLink()) { + return undefined; + } + return { dev: stat.dev, ino: stat.ino, mtimeMs: stat.mtimeMs, size: stat.size }; + } catch { + return undefined; + } +} + +export function legacyAuditRawCheckpointsMatch( + left: LegacyAuditFileCheckpoint | undefined, + right: LegacyAuditFileCheckpoint | undefined, +): boolean { + return ( + left !== undefined && + right !== undefined && + left.dev === right.dev && + left.ino === right.ino && + left.mtimeMs === right.mtimeMs && + left.size === right.size + ); +} + +function legacyAuditRawCheckpointIsCurrent( + sourcePath: string, + checkpoint: LegacyAuditRawCheckpoint, +): boolean { + let fd: number | undefined; + try { + fd = fs.openSync(sourcePath, "r"); + const beforeStat = fs.fstatSync(fd); + const before = { + dev: beforeStat.dev, + ino: beforeStat.ino, + mtimeMs: beforeStat.mtimeMs, + size: beforeStat.size, + }; + if (!beforeStat.isFile() || !legacyAuditRawCheckpointsMatch(checkpoint, before)) { + return false; + } + const hash = createHash("sha256"); + const chunk = Buffer.allocUnsafe(64 * 1024); + let offset = 0; + while (offset < checkpoint.size) { + const bytesRead = fs.readSync( + fd, + chunk, + 0, + Math.min(chunk.byteLength, checkpoint.size - offset), + offset, + ); + if (bytesRead === 0) { + return false; + } + hash.update(chunk.subarray(0, bytesRead)); + offset += bytesRead; + } + const afterStat = fs.fstatSync(fd); + const after = { + dev: afterStat.dev, + ino: afterStat.ino, + mtimeMs: afterStat.mtimeMs, + size: afterStat.size, + }; + return ( + legacyAuditRawCheckpointsMatch(before, after) && + offset === checkpoint.size && + hash.digest("hex") === checkpoint.contentHash + ); + } catch { + return false; + } finally { + if (fd !== undefined) { + fs.closeSync(fd); + } + } +} + +export function detectLegacyAuditLogs(params: { + stateDir: string; + doctorOnlyStateMigrations?: boolean; +}): LegacyAuditLogsDetection { + const logicalSources: Array> = [ + { + kind: "config", + label: "config audit log", + sourcePath: path.join(params.stateDir, "logs", "config-audit.jsonl"), + }, + { + kind: "system-agent", + label: "system-agent audit log", + sourcePath: path.join(params.stateDir, "audit", "system-agent.jsonl"), + }, + { + kind: "crestodian", + label: "Crestodian audit log", + sourcePath: path.join(params.stateDir, "audit", "crestodian.jsonl"), + }, + ]; + // Startup migrates every detected source. Retired audit imports belong only + // to explicit `doctor --fix` repair. + if (params.doctorOnlyStateMigrations !== true) { + return { sources: [], hasLegacy: false }; + } + let checkpoints: LegacyAuditRawCheckpoint[] | undefined; + const loadCheckpoints = () => { + if (checkpoints) { + return checkpoints; + } + try { + checkpoints = openLegacyAuditRawCheckpointStore(params.stateDir) + .entries() + .map((entry) => entry.value); + } catch { + checkpoints = []; + } + return checkpoints; + }; + const sources: LegacyAuditLogSource[] = []; + for (const logical of logicalSources) { + let directoryEntries: string[] = []; + try { + directoryEntries = fs.readdirSync(path.dirname(logical.sourcePath)); + } catch { + // The active-path check below still preserves the ordinary detection result. + } + const baseName = path.basename(logical.sourcePath).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const rawArchivePattern = new RegExp( + `^${baseName}\\.migrated(?:\\.([2-9]|[1-9][0-9]+))?\\.raw$`, + "u", + ); + const claimPattern = new RegExp( + `^\\.${baseName}\\.doctor-importing(?:\\.([2-9]|[1-9][0-9]+))?$`, + "u", + ); + const rawArchives = directoryEntries + .flatMap((entry) => { + const match = rawArchivePattern.exec(entry); + return match ? [{ entry, generation: BigInt(match[1] ?? "1") }] : []; + }) + .toSorted( + (left, right) => + (left.generation < right.generation ? -1 : left.generation > right.generation ? 1 : 0) || + left.entry.localeCompare(right.entry), + ); + for (const { entry } of rawArchives) { + const rawPath = path.join(path.dirname(logical.sourcePath), entry); + const rawRelativePath = path.relative(path.resolve(params.stateDir), rawPath); + const generationKey = legacyAuditSourceGenerationKey(rawRelativePath); + const checkpoint = statLegacyAuditRawCheckpoint(rawPath); + const hasRecoveryJournal = + statLegacyAuditRawCheckpoint(`${rawPath}.doctor-scrub-restore`) !== undefined; + if ( + !hasRecoveryJournal && + checkpoint && + loadCheckpoints().some( + (candidate) => + candidate.generationKey === generationKey && + candidate.phase === "raw" && + candidate.recordCount === 0 && + legacyAuditRawCheckpointsMatch(candidate, checkpoint) && + legacyAuditRawCheckpointIsCurrent(rawPath, candidate), + ) + ) { + continue; + } + sources.push({ + ...logical, + sourcePath: rawPath, + logicalSourcePath: logical.sourcePath, + storage: "raw-archive", + sanitizedArchivePath: rawPath.slice(0, -".raw".length), + }); + } + // Claims reserve their archive generation across a crash. An older + // sanitized-only generation cannot be reused by a later claim. + const claims = directoryEntries + .flatMap((entry) => { + const match = claimPattern.exec(entry); + return match ? [{ entry, generation: BigInt(match[1] ?? "1") }] : []; + }) + .toSorted( + (left, right) => + (left.generation < right.generation ? -1 : left.generation > right.generation ? 1 : 0) || + left.entry.localeCompare(right.entry), + ); + for (const { entry, generation } of claims) { + const generationSuffix = generation === 1n ? "" : `.${generation}`; + const sanitizedArchivePath = `${logical.sourcePath}.migrated${generationSuffix}`; + sources.push({ + ...logical, + sourcePath: path.join(path.dirname(logical.sourcePath), entry), + logicalSourcePath: logical.sourcePath, + storage: "claim", + sanitizedArchivePath, + rawArchivePath: `${sanitizedArchivePath}.raw`, + }); + } + if (fs.existsSync(logical.sourcePath)) { + sources.push({ + ...logical, + logicalSourcePath: logical.sourcePath, + storage: "active", + }); + } + } + return { sources, hasLegacy: sources.length > 0 }; +} diff --git a/src/infra/state-migrations.audit-coordination.ts b/src/infra/state-migrations.audit-coordination.ts new file mode 100644 index 00000000000..13f36af58cc --- /dev/null +++ b/src/infra/state-migrations.audit-coordination.ts @@ -0,0 +1,25 @@ +import { withOpenClawStateLease } from "../state/openclaw-state-lease.js"; + +const LEGACY_AUDIT_COORDINATION_SCOPE = "migration.legacy-audit"; +const LEGACY_AUDIT_COORDINATION_KEY = "filesystem-sqlite-boundary"; + +export function withLegacyAuditMigrationLease( + stateDir: string, + run: () => Promise, +): Promise { + return withOpenClawStateLease( + { + scope: LEGACY_AUDIT_COORDINATION_SCOPE, + key: LEGACY_AUDIT_COORDINATION_KEY, + database: { + scope: "shared", + options: { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }, + }, + leaseMs: 60_000, + waitMs: 5_000, + leaseLabel: "legacy audit migration lease", + operationLabel: "migration.legacy-audit.lease", + }, + async () => await run(), + ); +} diff --git a/src/infra/state-migrations.audit-logs.test.ts b/src/infra/state-migrations.audit-logs.test.ts new file mode 100644 index 00000000000..f4d535b36f8 --- /dev/null +++ b/src/infra/state-migrations.audit-logs.test.ts @@ -0,0 +1,1042 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { CONFIG_AUDIT_MAX_ENTRIES, CONFIG_AUDIT_SCOPE } from "../config/io.audit.js"; +import { listConfigAuditRecordsForTests } from "../config/io.audit.test-support.js"; +import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; +import { SYSTEM_AGENT_AUDIT_SCOPE } from "../system-agent/audit.js"; +import { listSystemAgentAuditEntriesForTests } from "../system-agent/audit.test-support.js"; +import { withTempDir } from "../test-helpers/temp-dir.js"; +import { acquireGatewayLock } from "./gateway-lock.js"; +import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js"; +import { detectLegacyAuditLogs, migrateLegacyAuditLogs } from "./state-migrations.audit-logs.js"; + +const TEST_AUDIT_SCRUB_PATTERN = Buffer.from( + Array.from({ length: 32 }, (_, index) => (index % 2 === 0 ? 0x20 : 0x09)), +); + +function buildTestAuditScrubbedContent(length: number): Buffer { + const content = Buffer.allocUnsafe(length); + for (let offset = 0; offset < length; offset += TEST_AUDIT_SCRUB_PATTERN.length) { + TEST_AUDIT_SCRUB_PATTERN.copy( + content, + offset, + 0, + Math.min(TEST_AUDIT_SCRUB_PATTERN.length, length - offset), + ); + } + return content; +} + +async function buildTestAuditRestoreJournal( + rawPath: string, + sourceRaw: Buffer, + progress: { restoredBytes: number; scrubbedBytes: number } = { + restoredBytes: 0, + scrubbedBytes: 0, + }, +): Promise { + const stat = await fs.stat(rawPath); + const journal = `${JSON.stringify({ + schemaVersion: 6, + rawBase64: sourceRaw.toString("base64"), + scrubPatternBase64: TEST_AUDIT_SCRUB_PATTERN.toString("base64"), + target: { dev: stat.dev, ino: stat.ino, size: sourceRaw.length }, + })}\n`; + await fs.writeFile( + `${rawPath}.doctor-scrub-progress`, + `${JSON.stringify({ + schemaVersion: 1, + journalHash: createHash("sha256").update(journal).digest("hex"), + direction: progress.restoredBytes > 0 ? "restoring" : "scrubbing", + committedBytes: progress.restoredBytes > 0 ? progress.restoredBytes : progress.scrubbedBytes, + pendingEnd: progress.restoredBytes > 0 ? progress.restoredBytes : progress.scrubbedBytes, + extentBytes: progress.restoredBytes > 0 ? progress.scrubbedBytes : sourceRaw.length, + })}\n`, + ); + return journal; +} + +describe("legacy core audit log migration", () => { + afterEach(() => { + resetPluginStateStoreForTests(); + }); + + it("imports config and system audit JSONL only through explicit doctor repair", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-" }, async (stateDir) => { + const configPath = path.join(stateDir, "logs", "config-audit.jsonl"); + const systemPath = path.join(stateDir, "audit", "system-agent.jsonl"); + const crestodianPath = path.join(stateDir, "audit", "crestodian.jsonl"); + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.mkdir(path.dirname(systemPath), { recursive: true }); + const unredactedConfigRecord = { + ts: "2026-07-01T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "token", "must-redact"], + execArgv: [], + }; + const unredactedDigest = createHash("sha256") + .update(JSON.stringify(unredactedConfigRecord)) + .digest("hex") + .slice(0, 16); + await fs.writeFile(configPath, `${JSON.stringify(unredactedConfigRecord)}\n`); + await fs.writeFile( + systemPath, + `${JSON.stringify({ + timestamp: "2026-07-02T00:00:00.000Z", + operation: "config.set", + summary: "Set config", + })}\n`, + ); + await fs.writeFile( + crestodianPath, + `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Restarted gateway", + })}\n`, + ); + + expect(detectLegacyAuditLogs({ stateDir }).hasLegacy).toBe(false); + const detected = detectLegacyAuditLogs({ + stateDir, + doctorOnlyStateMigrations: true, + }); + expect(detected.sources).toHaveLength(3); + + const result = await migrateLegacyAuditLogs({ detected, stateDir }); + expect(result.warnings).toEqual([]); + expect(result.changes).toHaveLength(6); + + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + const configRecords = listConfigAuditRecordsForTests({ env, homedir: () => stateDir }); + expect(configRecords).toHaveLength(1); + expect(JSON.stringify(configRecords)).not.toContain("must-redact"); + const configEntries = createSqliteAuditRecordStore({ + scope: CONFIG_AUDIT_SCOPE, + maxEntries: CONFIG_AUDIT_MAX_ENTRIES, + env, + }).entries(); + expect(configEntries[0]?.key).not.toContain(unredactedDigest); + const archivedConfig = await fs.readFile(`${configPath}.migrated`, "utf8"); + expect(archivedConfig).not.toContain("must-redact"); + const rawArchivedConfig = await fs.readFile(`${configPath}.migrated.raw`, "utf8"); + expect(rawArchivedConfig).not.toContain("must-redact"); + expect(rawArchivedConfig.trim()).toBe(""); + if (process.platform !== "win32") { + expect((await fs.stat(`${configPath}.migrated`)).mode & 0o777).toBe(0o600); + expect((await fs.stat(`${configPath}.migrated.raw`)).mode & 0o777).toBe(0o600); + } + expect(JSON.parse(archivedConfig.trim())).toMatchObject({ + argv: ["openclaw", "config", "set", "token", "***"], + }); + expect( + listSystemAgentAuditEntriesForTests({ env }) + .map((entry) => entry.value.operation) + .toSorted(), + ).toEqual(["config.set", "gateway.restart"]); + await expect(fs.access(configPath)).rejects.toThrow(); + await expect(fs.access(systemPath)).rejects.toThrow(); + await expect(fs.access(crestodianPath)).rejects.toThrow(); + + expect(detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).hasLegacy).toBe( + false, + ); + const laterConfigRecord = { + ...unredactedConfigRecord, + ts: "2026-07-04T00:00:00.000Z", + argv: ["openclaw", "config", "set", "token", "later-redaction-marker"], + }; + await fs.appendFile(`${configPath}.migrated.raw`, `${JSON.stringify(laterConfigRecord)}\n`); + const rawRecovery = detectLegacyAuditLogs({ + stateDir, + doctorOnlyStateMigrations: true, + }); + expect(rawRecovery.sources).toMatchObject([{ storage: "raw-archive" }]); + const recovered = await migrateLegacyAuditLogs({ detected: rawRecovery, stateDir }); + expect(recovered.warnings).toEqual([]); + expect(recovered.changes.join("\n")).toContain("Recovered 1 later config audit log row"); + expect(listConfigAuditRecordsForTests({ env, homedir: () => stateDir })).toHaveLength(2); + await expect(fs.readFile(`${configPath}.migrated`, "utf8")).resolves.not.toContain( + "later-redaction-marker", + ); + await expect(fs.readFile(`${configPath}.migrated.raw`, "utf8")).resolves.not.toContain( + "later-redaction-marker", + ); + const recoveredArchiveRows = (await fs.readFile(`${configPath}.migrated`, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { ts: string }); + expect(recoveredArchiveRows.map((row) => row.ts)).toEqual([ + "2026-07-01T00:00:00.000Z", + "2026-07-04T00:00:00.000Z", + ]); + expect(detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).hasLegacy).toBe( + false, + ); + }); + }); + + it("rehashes a checkpointed raw archive before treating it as clean", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-rehash-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const original = { + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "original", + }; + const modified = { ...original, summary: "modified" }; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, `${JSON.stringify(original)}\n`); + const stableMtime = new Date("2026-07-03T01:00:00.000Z"); + await fs.utimes(sourcePath, stableMtime, stableMtime); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + const checkpointedStat = await fs.stat(rawPath); + const modifiedRaw = `${JSON.stringify(modified)}\n`; + expect(Buffer.byteLength(modifiedRaw)).toBe(checkpointedStat.size); + await fs.writeFile(rawPath, modifiedRaw); + await fs.utimes(rawPath, checkpointedStat.atimeMs / 1_000, checkpointedStat.mtimeMs / 1_000); + const rewrittenStat = await fs.stat(rawPath); + expect(rewrittenStat.mtimeMs).toBe(checkpointedStat.mtimeMs); + expect(rewrittenStat.size).toBe(checkpointedStat.size); + + const detected = detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }); + expect(detected.sources).toMatchObject([{ sourcePath: rawPath, storage: "raw-archive" }]); + const result = await migrateLegacyAuditLogs({ detected, stateDir }); + + expect(result.warnings.join("\n")).toContain("changed other than by append"); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["original"]); + }); + }); + + it("restores the captured archive prefix when an in-place scrub write fails", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-scrub-write-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const originalRecord = { + ts: "2026-07-01T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "token", "scrub-write-marker"], + execArgv: [], + }; + const originalContent = `${JSON.stringify(originalRecord)}\n`; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, originalContent); + + const probe = await fs.open(path.join(stateDir, "write-probe"), "w+"); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + write( + buffer: Uint8Array, + offset: number, + length: number, + position: number | null, + ): Promise<{ bytesWritten: number; buffer: Uint8Array }>; + }; + await probe.close(); + const originalWrite = Reflect.get( + fileHandlePrototype, + "write", + ) as typeof fileHandlePrototype.write; + let scrubWriteCalls = 0; + const writeSpy = vi.spyOn(fileHandlePrototype, "write").mockImplementation(async function ( + this: typeof fileHandlePrototype, + buffer: Uint8Array, + offset: number, + length: number, + position: number | null, + ) { + scrubWriteCalls += 1; + if (scrubWriteCalls === 1) { + return await originalWrite.call( + this, + buffer, + offset, + Math.max(1, Math.floor(length / 2)), + position, + ); + } + if (scrubWriteCalls === 2) { + throw new Error("simulated scrub write failure"); + } + return await originalWrite.call(this, buffer, offset, length, position); + }); + + let failed: Awaited>; + try { + failed = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + } finally { + writeSpy.mockRestore(); + } + + expect(failed.warnings.join("\n")).toContain("restored it for Doctor retry"); + await expect(fs.readFile(rawPath, "utf8")).resolves.toBe(originalContent); + expect(detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).hasLegacy).toBe( + true, + ); + + const recovered = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + expect(recovered.warnings).toEqual([]); + await expect(fs.readFile(rawPath, "utf8")).resolves.not.toContain("scrub-write-marker"); + expect(detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).hasLegacy).toBe( + false, + ); + }); + }); + + it("restores a moved scrub journal before parsing an interrupted archive", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-scrub-restart-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl"); + const sanitizedPath = `${sourcePath}.migrated`; + const rawPath = `${sanitizedPath}.raw`; + const restorePath = `${rawPath}.doctor-scrub-restore`; + const originalRecord = { + ts: "2026-07-01T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "token", "restart-redaction-marker"], + execArgv: [], + }; + const originalContent = `${JSON.stringify(originalRecord)}\n`; + const damagedContent = Buffer.from(originalContent, "utf8"); + buildTestAuditScrubbedContent(damagedContent.length) + .subarray(0, Math.floor(damagedContent.length / 2)) + .copy(damagedContent); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sanitizedPath, "{}\n"); + await fs.writeFile(rawPath, damagedContent); + await fs.writeFile( + restorePath, + await buildTestAuditRestoreJournal(rawPath, Buffer.from(originalContent, "utf8"), { + restoredBytes: 0, + scrubbedBytes: Math.floor(damagedContent.length / 2), + }), + ); + + const result = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(result.warnings).toEqual([]); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.readFile(rawPath, "utf8")).resolves.not.toContain("restart-redaction-marker"); + expect( + listConfigAuditRecordsForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + homedir: () => stateDir, + }), + ).toHaveLength(1); + expect(detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).hasLegacy).toBe( + false, + ); + }); + }); + + it("discards a stale restore journal while recovering a post-checkpoint append", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-stale-journal-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const restorePath = `${rawPath}.doctor-scrub-restore`; + const originalContent = `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "config.set", + summary: ["token", "redaction-marker"].join("="), + })}\n`; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, originalContent); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + await fs.writeFile( + restorePath, + await buildTestAuditRestoreJournal(rawPath, Buffer.from(originalContent, "utf8")), + ); + await fs.appendFile( + rawPath, + `${JSON.stringify({ + timestamp: "2026-07-04T00:00:00.000Z", + operation: "gateway.restart", + summary: "later", + })}\n`, + ); + + const detected = detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }); + expect(detected.hasLegacy).toBe(true); + const result = await migrateLegacyAuditLogs({ detected, stateDir }); + + expect(result.warnings).toEqual([]); + expect(result.changes.join("\n")).toContain("Recovered 1 later"); + await expect(fs.readFile(rawPath, "utf8")).resolves.not.toContain( + ["token", "redaction-marker"].join("="), + ); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); + + it("does not replay a scrub journal over a same-inode replacement archive", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-scrub-replaced-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const sanitizedPath = `${sourcePath}.migrated`; + const rawPath = `${sanitizedPath}.raw`; + const restorePath = `${rawPath}.doctor-scrub-restore`; + const originalContent = `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "original archive", + })}\n`; + const replacementContent = `${JSON.stringify({ + timestamp: "2026-07-04T00:00:00.000Z", + operation: "gateway.restart", + summary: "replacement archive", + })}\n`; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sanitizedPath, "{}\n"); + await fs.writeFile(rawPath, originalContent); + await fs.writeFile( + restorePath, + await buildTestAuditRestoreJournal(rawPath, Buffer.from(originalContent, "utf8")), + ); + await fs.writeFile(rawPath, replacementContent); + + const result = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(result.warnings.join("\n")).toContain("no longer matches its restore journal target"); + await expect(fs.readFile(rawPath, "utf8")).resolves.toBe(replacementContent); + await expect(fs.access(restorePath)).resolves.toBeUndefined(); + }); + }); + + it("resumes a deterministic audit claim left by an interrupted Doctor", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-resume-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const claimPath = path.join( + path.dirname(sourcePath), + `.${path.basename(sourcePath)}.doctor-importing`, + ); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile( + sourcePath, + `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Restarted gateway", + })}\n`, + ); + await fs.rename(sourcePath, claimPath); + + const detected = detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }); + expect(detected.sources).toMatchObject([{ sourcePath: claimPath, storage: "claim" }]); + const result = await migrateLegacyAuditLogs({ detected, stateDir }); + + expect(result.warnings).toEqual([]); + await expect(fs.access(claimPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.access(`${sourcePath}.migrated.raw`)).resolves.toBeUndefined(); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toHaveLength(1); + + const tenthRawArchive = `${sourcePath}.migrated.10.raw`; + await fs.writeFile( + tenthRawArchive, + `${JSON.stringify({ + timestamp: "2026-07-04T00:00:00.000Z", + operation: "gateway.reload", + summary: "Reloaded gateway", + })}\n`, + ); + const numberedRaw = detectLegacyAuditLogs({ + stateDir, + doctorOnlyStateMigrations: true, + }); + expect(numberedRaw.sources).toMatchObject([ + { sourcePath: tenthRawArchive, storage: "raw-archive" }, + ]); + await migrateLegacyAuditLogs({ detected: numberedRaw, stateDir }); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toHaveLength(2); + }); + }); + + it("does not resurrect a pruned raw-archive head when later rows are appended", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-late-tail-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const records = ["first", "second", "third"].map((summary, index) => ({ + timestamp: `2026-07-03T00:00:0${index}.000Z`, + operation: "gateway.restart", + summary, + })); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile( + sourcePath, + `${records.map((record) => JSON.stringify(record)).join("\n")}\n`, + ); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + createSqliteAuditRecordStore({ + scope: SYSTEM_AGENT_AUDIT_SCOPE, + maxEntries: 3, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).register("runtime", { + timestamp: "2026-07-04T00:00:00.000Z", + operation: "gateway.reload", + summary: "runtime", + }); + await fs.appendFile( + `${sourcePath}.migrated.raw`, + `${JSON.stringify({ + timestamp: "2026-07-05T00:00:00.000Z", + operation: "gateway.restart", + summary: "appended", + })}\n`, + ); + + const recovered = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(recovered.warnings).toEqual([]); + expect(recovered.changes.join("\n")).toContain("Recovered 1 later"); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["second", "third", "appended", "runtime"]); + const rawCheckpoints = createSqliteAuditRecordStore<{ recordCount: number }>({ + scope: "migration.legacy-audit-raw", + maxEntries: 10_000, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).entries(); + expect(rawCheckpoints).toHaveLength(1); + expect(rawCheckpoints[0]?.value.recordCount).toBe(0); + }); + }); + + it("keeps identical rows from separate raw archive generations", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-generations-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const record = { + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Repeated operation", + }; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(`${sourcePath}.migrated.raw`, `${JSON.stringify(record)}\n`); + await fs.writeFile(`${sourcePath}.migrated.2.raw`, `${JSON.stringify(record)}\n`); + await fs.writeFile(`${sourcePath}.migrated.10.raw`, `${JSON.stringify(record)}\n`); + const claimPath = path.join( + path.dirname(sourcePath), + `.${path.basename(sourcePath)}.doctor-importing.11`, + ); + await fs.writeFile(claimPath, `${JSON.stringify(record)}\n`); + await fs.writeFile(sourcePath, `${JSON.stringify(record)}\n`); + + const detected = detectLegacyAuditLogs({ + stateDir, + doctorOnlyStateMigrations: true, + }); + expect(detected.sources.map((source) => path.basename(source.sourcePath))).toEqual([ + "system-agent.jsonl.migrated.raw", + "system-agent.jsonl.migrated.2.raw", + "system-agent.jsonl.migrated.10.raw", + ".system-agent.jsonl.doctor-importing.11", + "system-agent.jsonl", + ]); + const result = await migrateLegacyAuditLogs({ detected, stateDir }); + + expect(result.warnings).toEqual([]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toHaveLength(5); + expect(detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).hasLegacy).toBe( + false, + ); + }); + }); + + it("keeps restored raw archives idempotent after device and inode changes", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-restored-" }, async (rootDir) => { + const stateDir = path.join(rootDir, "source-state"); + const restoredStateDir = path.join(rootDir, "restored-state"); + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile( + sourcePath, + `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Restored operation", + })}\n`, + ); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + resetPluginStateStoreForTests(); + await fs.cp(stateDir, restoredStateDir, { recursive: true }); + createSqliteAuditRecordStore({ + scope: SYSTEM_AGENT_AUDIT_SCOPE, + maxEntries: 1, + env: { ...process.env, OPENCLAW_STATE_DIR: restoredStateDir }, + }).register("runtime-after-restore", { + timestamp: "2026-07-04T00:00:00.000Z", + operation: "gateway.reload", + summary: "Runtime after restore", + }); + + const restored = detectLegacyAuditLogs({ + stateDir: restoredStateDir, + doctorOnlyStateMigrations: true, + }); + expect(restored.sources).toMatchObject([{ storage: "raw-archive" }]); + const result = await migrateLegacyAuditLogs({ + detected: restored, + stateDir: restoredStateDir, + }); + + expect(result.warnings).toEqual([]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: restoredStateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["Runtime after restore"]); + expect( + detectLegacyAuditLogs({ + stateDir: restoredStateDir, + doctorOnlyStateMigrations: true, + }).hasLegacy, + ).toBe(false); + }); + }); + + it("resumes the stable generation of an interrupted sanitized archive", async () => { + await withTempDir( + { prefix: "openclaw-audit-migration-sanitized-resume-" }, + async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const claimPath = path.join( + path.dirname(sourcePath), + `.${path.basename(sourcePath)}.doctor-importing`, + ); + const record = { + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Interrupted operation", + }; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(claimPath, `${JSON.stringify(record)}\n`); + await fs.writeFile(`${sourcePath}.migrated`, `${JSON.stringify(record)}\n`); + + const result = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(result.warnings).toEqual([]); + await expect(fs.access(`${sourcePath}.migrated.raw`)).resolves.toBeUndefined(); + await expect(fs.access(`${sourcePath}.migrated.2.raw`)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toHaveLength(1); + }, + ); + }); + + it("allocates a new generation when an active source follows a sanitized-only archive", async () => { + await withTempDir( + { prefix: "openclaw-audit-migration-sanitized-recreated-" }, + async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const record = { + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Repeated after downgrade", + }; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, `${JSON.stringify(record)}\n`); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + await fs.rm(`${sourcePath}.migrated.raw`); + const firstSanitized = await fs.readFile(`${sourcePath}.migrated`, "utf8"); + await fs.writeFile(sourcePath, `${JSON.stringify(record)}\n`); + + const repeated = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(repeated.warnings).toEqual([]); + expect(repeated.changes.join("\n")).toContain("1 new row"); + await expect(fs.readFile(`${sourcePath}.migrated`, "utf8")).resolves.toBe(firstSanitized); + await expect(fs.access(`${sourcePath}.migrated.2.raw`)).resolves.toBeUndefined(); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toHaveLength(2); + }, + ); + }); + + it("resumes a claim at its reserved generation instead of an older sanitized-only slot", async () => { + await withTempDir( + { prefix: "openclaw-audit-migration-claimed-generation-" }, + async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const secondClaimPath = path.join( + path.dirname(sourcePath), + `.${path.basename(sourcePath)}.doctor-importing.2`, + ); + const record = { + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Repeated after interrupted downgrade", + }; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, `${JSON.stringify(record)}\n`); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + await fs.rm(`${sourcePath}.migrated.raw`); + await fs.writeFile(sourcePath, `${JSON.stringify(record)}\n`); + await fs.rename(sourcePath, secondClaimPath); + + const detected = detectLegacyAuditLogs({ + stateDir, + doctorOnlyStateMigrations: true, + }); + expect(detected.sources).toMatchObject([ + { + sourcePath: secondClaimPath, + storage: "claim", + sanitizedArchivePath: `${sourcePath}.migrated.2`, + rawArchivePath: `${sourcePath}.migrated.2.raw`, + }, + ]); + const resumed = await migrateLegacyAuditLogs({ detected, stateDir }); + + expect(resumed.warnings).toEqual([]); + await expect(fs.access(`${sourcePath}.migrated.2.raw`)).resolves.toBeUndefined(); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toHaveLength(2); + }, + ); + }); + + it("leaves malformed audit sources in place without partial imports", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-invalid-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, "{bad json\n"); + const detected = detectLegacyAuditLogs({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyAuditLogs({ detected, stateDir }); + expect(result.warnings.join("\n")).toContain("Failed reading system-agent audit log"); + await expect(fs.access(sourcePath)).resolves.toBeUndefined(); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toEqual([]); + }); + }); + + it("does not migrate newer audit generations before an older source is repaired", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-order-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const event = (summary: string) => ({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary, + }); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(rawPath, "{bad json\n"); + await fs.writeFile(sourcePath, `${JSON.stringify(event("newer generation"))}\n`); + + const blocked = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(blocked.changes).toEqual([]); + expect(blocked.warnings.join("\n")).toContain("Failed reading system-agent audit log"); + await expect(fs.access(sourcePath)).resolves.toBeUndefined(); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toEqual([]); + + await fs.writeFile(rawPath, `${JSON.stringify(event("repaired older generation"))}\n`); + const repaired = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(repaired.warnings).toEqual([]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["repaired older generation", "newer generation"]); + }); + }); + + it("restores the active source when raw archive hardening fails", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-permissions-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl"); + const record = { + ts: "2026-07-01T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "token", "must-redact"], + execArgv: [], + }; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, `${JSON.stringify(record)}\n`); + const probe = await fs.open(path.join(stateDir, "chmod-probe"), "w"); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + chmod(mode: number): Promise; + }; + await probe.close(); + const originalChmod = Reflect.get(fileHandlePrototype, "chmod") as ( + this: typeof fileHandlePrototype, + mode: number, + ) => Promise; + let chmodCalls = 0; + const chmodSpy = vi.spyOn(fileHandlePrototype, "chmod").mockImplementation(function ( + this: typeof fileHandlePrototype, + mode: number, + ) { + chmodCalls += 1; + if (chmodCalls === 3) { + return Promise.reject(new Error("simulated chmod failure")); + } + return originalChmod.call(this, mode); + }); + + let failed: Awaited>; + try { + failed = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + } finally { + chmodSpy.mockRestore(); + } + + expect(failed.changes).toEqual([]); + expect(failed.warnings.join("\n")).toContain("Failed securing raw archived config audit log"); + await expect(fs.readFile(sourcePath, "utf8")).resolves.toContain("must-redact"); + await expect(fs.access(`${sourcePath}.migrated`)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.access(`${sourcePath}.migrated.raw`)).rejects.toMatchObject({ + code: "ENOENT", + }); + + const recovered = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + expect(recovered.warnings).toEqual([]); + await expect(fs.access(sourcePath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.access(`${sourcePath}.migrated.raw`)).resolves.toBeUndefined(); + }); + }); + + it("requires exclusive state ownership before claiming legacy audit files", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-lock-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile( + sourcePath, + `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Restarted gateway", + })}\n`, + ); + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + const gatewayLock = await acquireGatewayLock({ + allowInTests: true, + env, + pollIntervalMs: 10, + port: 18_791, + timeoutMs: 100, + }); + if (!gatewayLock) { + throw new Error("expected test Gateway lock"); + } + + let result: Awaited>; + try { + result = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + } finally { + await gatewayLock.release(); + } + + expect(result.warnings.join("\n")).toContain("exclusive state ownership is unavailable"); + await expect(fs.access(sourcePath)).resolves.toBeUndefined(); + expect(listSystemAgentAuditEntriesForTests({ env })).toEqual([]); + }); + }); + + it("leaves rows from an old config writer at the recreated source path", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-concurrent-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl"); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + const records = Array.from({ length: 10_000 }, (_, index) => + JSON.stringify({ + ts: new Date(Date.UTC(2026, 6, 1, 0, 0, index)).toISOString(), + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", `key-${index}`, "value"], + execArgv: [], + }), + ); + await fs.writeFile(sourcePath, `${records.join("\n")}\n`); + const retainedRecord = { + ts: "2026-07-02T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "later", "value"], + execArgv: [], + }; + + let settled = false; + const migration = migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }).finally(() => { + settled = true; + }); + for (let attempt = 0; attempt < 500; attempt += 1) { + try { + await fs.access(sourcePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + break; + } + throw error; + } + await new Promise((resolve) => { + setImmediate(resolve); + }); + } + expect(settled).toBe(false); + await fs.appendFile(sourcePath, `${JSON.stringify(retainedRecord)}\n`); + + const first = await migration; + expect(first.warnings.join("\n")).toContain("An old writer recreated config audit log"); + await expect(fs.readFile(sourcePath, "utf8")).resolves.toBe( + `${JSON.stringify(retainedRecord)}\n`, + ); + + const second = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + expect(second.warnings).toEqual([]); + await expect(fs.access(sourcePath)).rejects.toMatchObject({ code: "ENOENT" }); + expect( + createSqliteAuditRecordStore({ + scope: CONFIG_AUDIT_SCOPE, + maxEntries: CONFIG_AUDIT_MAX_ENTRIES, + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).entries(), + ).toHaveLength(10_001); + }); + }); + + it.runIf(process.platform !== "win32")( + "rejects audit sources beneath symlinked state parents", + async () => { + const externalAuditDir = await fs.mkdtemp( + path.join(os.tmpdir(), "openclaw-audit-migration-external-"), + ); + try { + await withTempDir({ prefix: "openclaw-audit-migration-symlink-" }, async (stateDir) => { + const externalSource = path.join(externalAuditDir, "system-agent.jsonl"); + await fs.writeFile( + externalSource, + `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Outside state root", + })}\n`, + ); + await fs.symlink(externalAuditDir, path.join(stateDir, "audit")); + const detected = detectLegacyAuditLogs({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyAuditLogs({ detected, stateDir }); + + expect(result.changes).toEqual([]); + expect(result.warnings.join("\n")).toMatch(/alias|symlink|outside workspace/u); + await expect(fs.readFile(externalSource, "utf8")).resolves.toContain( + "Outside state root", + ); + await expect(fs.access(`${externalSource}.migrated`)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toEqual([]); + }); + } finally { + await fs.rm(externalAuditDir, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/src/infra/state-migrations.audit-logs.ts b/src/infra/state-migrations.audit-logs.ts new file mode 100644 index 00000000000..af95fff4bdd --- /dev/null +++ b/src/infra/state-migrations.audit-logs.ts @@ -0,0 +1,669 @@ +// Doctor-only import for retired core JSONL audit stores. +import fs from "node:fs"; +import path from "node:path"; +import { + CONFIG_AUDIT_MAX_ENTRIES, + CONFIG_AUDIT_SCOPE, + type ConfigAuditRecord, +} from "../config/io.audit.js"; +import { + SYSTEM_AGENT_AUDIT_MAX_ENTRIES, + SYSTEM_AGENT_AUDIT_SCOPE, + type SystemAgentAuditEntry, +} from "../system-agent/audit.js"; +import { root as createFsSafeRoot } from "./fs-safe.js"; +import { acquireGatewayLock } from "./gateway-lock.js"; +import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js"; +import { + hasLegacyAuditRawCheckpointCapacity, + legacyAuditSourceGenerationKey, +} from "./state-migrations.audit-checkpoints.js"; +import { withLegacyAuditMigrationLease } from "./state-migrations.audit-coordination.js"; +import type { + LegacyAuditLogSource, + LegacyAuditLogsDetection, +} from "./state-migrations.audit-logs.types.js"; +import { + prepareLegacyAuditRecords, + serializePreparedAuditRecords, + type PreparedAuditRecord, +} from "./state-migrations.audit-records.js"; +import { + finalizeLegacyAuditRecoveryArchive, + findPreviousLegacyAuditRawCheckpoint, + readLegacyAuditSourceSnapshot, + recordLegacyAuditRawCheckpoint, + recordsAfterLegacyAuditRawCheckpoint, + restoreInterruptedAuditRecoveryArchive, + scrubLegacyAuditRecoveryArchive, + type AuditMigrationRoot, + type LegacyAuditSourceSnapshot, +} from "./state-migrations.audit-recovery.js"; +import { writeRecoveredSanitizedAuditArchive } from "./state-migrations.audit-sanitized.js"; +import type { MigrationMessages } from "./state-migrations.types.js"; + +function legacyAuditClaimPathForArchive(sourcePath: string, sanitizedArchivePath: string): string { + const archivePrefix = `${sourcePath}.migrated`; + if (!sanitizedArchivePath.startsWith(archivePrefix)) { + throw new Error(`Invalid legacy audit archive path ${sanitizedArchivePath}`); + } + const generationSuffix = sanitizedArchivePath.slice(archivePrefix.length); + return path.join( + path.dirname(sourcePath), + `.${path.basename(sourcePath)}.doctor-importing${generationSuffix}`, + ); +} + +export { detectLegacyAuditLogs } from "./state-migrations.audit-checkpoints.js"; + +type AuditArchiveRelativePaths = { + sanitized: string; + raw: string; + resumeSanitized: boolean; +}; + +async function resolveAuditArchiveRelativePaths( + root: AuditMigrationRoot, + sourceRelativePath: string, +): Promise { + const directoryPath = path.join(root.rootReal, path.dirname(sourceRelativePath)); + const baseName = path.basename(sourceRelativePath).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const archivePattern = new RegExp( + `^${baseName}\\.migrated(?:\\.([2-9]|[1-9][0-9]+))?(?:\\.raw)?$`, + "u", + ); + const claimPattern = new RegExp( + `^\\.${baseName}\\.doctor-importing(?:\\.([2-9]|[1-9][0-9]+))?$`, + "u", + ); + let latestGeneration = 0n; + for (const entry of fs.readdirSync(directoryPath)) { + const match = archivePattern.exec(entry) ?? claimPattern.exec(entry); + if (!match) { + continue; + } + const generation = BigInt(match[1] ?? "1"); + if (generation > latestGeneration) { + latestGeneration = generation; + } + } + const generation = latestGeneration + 1n; + const sanitized = `${sourceRelativePath}.migrated${generation === 1n ? "" : `.${generation}`}`; + return { sanitized, raw: `${sanitized}.raw`, resumeSanitized: false }; +} + +async function secureAuditArchiveFile(params: { + root: AuditMigrationRoot; + relativePath: string; + label: string; + warnings: string[]; +}): Promise { + try { + const opened = await params.root.open(params.relativePath); + try { + await opened.handle.chmod(0o600); + await opened.handle.sync(); + } finally { + await opened.handle.close(); + } + return true; + } catch (error) { + params.warnings.push(`Failed securing ${params.label} legacy source: ${String(error)}`); + return false; + } +} + +async function archiveLegacyAuditClaim(params: { + source: LegacyAuditLogSource; + claimRelativePath: string; + archivePaths: { sanitized: string; raw: string; resumeSanitized: boolean }; + snapshot: LegacyAuditSourceSnapshot; + sanitizedJsonl: string; + root: AuditMigrationRoot; + changes: string[]; + warnings: string[]; +}): Promise<{ + moved: boolean; + rawRelativePath?: string; + scrubbedSnapshot?: LegacyAuditSourceSnapshot; +}> { + let moved = false; + let sanitizedCreated = false; + const archivePaths = params.archivePaths; + try { + if (archivePaths.resumeSanitized) { + await params.root.write(archivePaths.sanitized, params.sanitizedJsonl, { + mkdir: false, + mode: 0o600, + }); + } else { + await params.root.create(archivePaths.sanitized, params.sanitizedJsonl, { mode: 0o600 }); + } + sanitizedCreated = true; + if ( + !(await secureAuditArchiveFile({ + root: params.root, + relativePath: archivePaths.sanitized, + label: `sanitized ${params.source.label}`, + warnings: params.warnings, + })) + ) { + return { moved: false }; + } + // Keep the claimed inode intact. A predecessor CLI may already hold an append + // descriptor across the claim; moving that inode to a named migration backup + // preserves any late write while the sanitized sibling remains safe to inspect. + await params.root.move(params.claimRelativePath, archivePaths.raw); + if ( + !(await secureAuditArchiveFile({ + root: params.root, + relativePath: archivePaths.raw, + label: `raw archived ${params.source.label}`, + warnings: params.warnings, + })) + ) { + try { + await params.root.move(archivePaths.raw, params.claimRelativePath); + } catch (error) { + params.warnings.push( + `Failed restoring unsecured ${params.source.label} legacy source: ${String(error)}`, + ); + } + return { moved: false }; + } + moved = true; + const scrubbedSnapshot = await scrubLegacyAuditRecoveryArchive({ + root: params.root, + relativePath: archivePaths.raw, + expectedSnapshot: params.snapshot, + label: params.source.label, + warnings: params.warnings, + }); + params.changes.push( + `Archived sanitized ${params.source.label} legacy source → ${path.join(path.dirname(params.source.logicalSourcePath), path.basename(archivePaths.sanitized))}; ${scrubbedSnapshot ? "scrubbed same-inode append recovery archive" : "retained same-inode append recovery archive for Doctor retry"} → ${path.join(path.dirname(params.source.logicalSourcePath), path.basename(archivePaths.raw))}`, + ); + return { + moved: true, + rawRelativePath: archivePaths.raw, + ...(scrubbedSnapshot ? { scrubbedSnapshot } : {}), + }; + } catch (error) { + params.warnings.push( + `Failed archiving ${params.source.label} ${params.source.logicalSourcePath}: ${String(error)}`, + ); + } finally { + if (!moved && sanitizedCreated) { + await params.root.remove(archivePaths.sanitized).catch(() => undefined); + } + } + return { moved, ...(moved ? { rawRelativePath: archivePaths.raw } : {}) }; +} + +async function restoreOrPreserveLegacyAuditClaim(params: { + source: LegacyAuditLogSource; + claimRelativePath: string; + sourceRelativePath: string; + archivePaths: AuditArchiveRelativePaths; + root: AuditMigrationRoot; + warnings: string[]; +}): Promise { + try { + if (!(await params.root.exists(params.claimRelativePath))) { + return; + } + if (!(await params.root.exists(params.sourceRelativePath))) { + await params.root.move(params.claimRelativePath, params.sourceRelativePath); + await secureAuditArchiveFile({ + root: params.root, + relativePath: params.sourceRelativePath, + label: params.source.label, + warnings: params.warnings, + }); + return; + } + await params.root.move(params.claimRelativePath, params.archivePaths.raw); + await secureAuditArchiveFile({ + root: params.root, + relativePath: params.archivePaths.raw, + label: `preserved ${params.source.label}`, + warnings: params.warnings, + }); + params.warnings.push( + `Preserved claimed ${params.source.label} at ${path.join(path.dirname(params.source.logicalSourcePath), path.basename(params.archivePaths.raw))} because an old writer recreated ${params.source.logicalSourcePath}`, + ); + } catch (error) { + params.warnings.push( + `Failed restoring claimed ${params.source.label} ${params.source.logicalSourcePath}: ${String(error)}`, + ); + } +} + +async function migrateLegacyAuditLogSource(params: { + source: LegacyAuditLogSource; + stateDir: string; + recreatedSourceScheduled?: boolean; +}): Promise { + const changes: string[] = []; + const warnings: string[] = []; + const result = (completed: boolean) => ({ changes, warnings, completed }); + const root = await createFsSafeRoot(params.stateDir, { + hardlinks: "reject", + // Doctor previously accepted the complete legacy log; keep that migration + // contract while root operations enforce path and symlink boundaries. + maxBytes: Number.MAX_SAFE_INTEGER, + mkdir: false, + mode: 0o600, + symlinks: "reject", + }); + const sourceRelativePath = path.relative( + path.resolve(params.stateDir), + params.source.logicalSourcePath, + ); + const detectedRelativePath = path.relative( + path.resolve(params.stateDir), + params.source.sourcePath, + ); + let archivePaths: AuditArchiveRelativePaths | undefined; + let claimRelativePath = detectedRelativePath; + if (params.source.storage === "active") { + archivePaths = await resolveAuditArchiveRelativePaths(root, sourceRelativePath); + claimRelativePath = path.relative( + path.resolve(params.stateDir), + legacyAuditClaimPathForArchive( + params.source.logicalSourcePath, + path.join(params.stateDir, archivePaths.sanitized), + ), + ); + await root.move(detectedRelativePath, claimRelativePath); + } else if (params.source.storage === "claim") { + if (!params.source.sanitizedArchivePath || !params.source.rawArchivePath) { + throw new Error(`Missing reserved archive generation for ${params.source.sourcePath}`); + } + const sanitized = path.relative( + path.resolve(params.stateDir), + params.source.sanitizedArchivePath, + ); + const raw = path.relative(path.resolve(params.stateDir), params.source.rawArchivePath); + archivePaths = { + sanitized, + raw, + resumeSanitized: (await root.exists(sanitized)) && !(await root.exists(raw)), + }; + } + let claimFinalized = params.source.storage === "raw-archive"; + try { + if ( + !(await secureAuditArchiveFile({ + root, + relativePath: claimRelativePath, + label: `claimed ${params.source.label}`, + warnings, + })) + ) { + return result(false); + } + const rawArchiveRelativePath = archivePaths?.raw ?? detectedRelativePath; + if (!hasLegacyAuditRawCheckpointCapacity(params.stateDir, rawArchiveRelativePath)) { + warnings.push( + `Skipped ${params.source.label} migration because durable raw-archive checkpoint capacity is exhausted; left the legacy source in place`, + ); + return result(false); + } + if ( + !(await restoreInterruptedAuditRecoveryArchive({ + root, + relativePath: claimRelativePath, + label: params.source.label, + warnings, + })) + ) { + return result(false); + } + const snapshot = await readLegacyAuditSourceSnapshot(root, claimRelativePath); + const sourceGeneration = legacyAuditSourceGenerationKey(rawArchiveRelativePath); + const sanitizedRelativePath = + params.source.storage === "raw-archive" && params.source.sanitizedArchivePath + ? path.relative(path.resolve(params.stateDir), params.source.sanitizedArchivePath) + : undefined; + const previousCheckpoint = + params.source.storage === "raw-archive" + ? findPreviousLegacyAuditRawCheckpoint(params.stateDir, rawArchiveRelativePath) + : undefined; + if (params.source.storage === "raw-archive" && !previousCheckpoint) { + if (!sanitizedRelativePath) { + throw new Error(`Missing sanitized archive path for ${params.source.sourcePath}`); + } + const firstContentByte = snapshot.rawBytes.findIndex( + (byte) => byte !== 0x20 && byte !== 0x09 && byte !== 0x0a && byte !== 0x0d, + ); + if (snapshot.rawBytes.length > 0 && firstContentByte !== 0) { + warnings.push( + `Skipped ${params.source.label} recovery because its checkpointless raw archive begins with ambiguous whitespace; left the archive in place`, + ); + return result(false); + } + } + const prepared = prepareLegacyAuditRecords( + params.source, + snapshot.raw, + sourceGeneration, + previousCheckpoint?.recordOrdinalBase ?? 0, + ); + if (!prepared.ok) { + warnings.push(...prepared.warnings); + return result(false); + } + const env = { ...process.env, OPENCLAW_STATE_DIR: params.stateDir }; + const maxEntries = + params.source.kind === "config" ? CONFIG_AUDIT_MAX_ENTRIES : SYSTEM_AGENT_AUDIT_MAX_ENTRIES; + const store = createSqliteAuditRecordStore({ + scope: params.source.kind === "config" ? CONFIG_AUDIT_SCOPE : SYSTEM_AGENT_AUDIT_SCOPE, + maxEntries, + env, + }); + const existingEntries = store.entries(); + const existingKeys = new Set(existingEntries.map((entry) => entry.key)); + let candidateRecords: readonly PreparedAuditRecord[] = prepared.records; + if (params.source.storage === "raw-archive") { + if (previousCheckpoint) { + const appendedRecords = recordsAfterLegacyAuditRawCheckpoint({ + checkpoint: previousCheckpoint, + snapshot, + records: prepared.records, + }); + if (!appendedRecords) { + warnings.push( + `Skipped ${params.source.label} recovery because ${params.source.sourcePath} changed other than by append; left the raw archive in place`, + ); + return result(false); + } + candidateRecords = appendedRecords; + } + } + if (!previousCheckpoint && candidateRecords === prepared.records) { + const lastRetainedSourceIndex = prepared.records.findLastIndex((record) => + existingKeys.has(record.key), + ); + if (lastRetainedSourceIndex >= 0) { + // A crash can occur after bounded insertion but before its raw checkpoint. + // Continue after the latest retained source ordinal instead of resurrecting + // the pruned head as newly appended audit history. + candidateRecords = prepared.records.slice(lastRetainedSourceIndex + 1); + } + } + const missing = candidateRecords.filter((record) => !existingKeys.has(record.key)); + // SQLite may commit before the sanitized merge/checkpoint. Keep raw-derived + // candidates intact so retry can still prove and materialize the artifact tail. + store.registerLegacyMany(missing); + const importedKeys = new Set(store.entries().map((entry) => entry.key)); + const retainedNewRows = missing.filter((record) => importedKeys.has(record.key)).length; + const retentionNote = + retainedNewRows === missing.length + ? "" + : `; ${retainedNewRows} retained after bounded retention`; + if (params.source.storage === "raw-archive") { + if (!sanitizedRelativePath) { + throw new Error(`Missing sanitized archive path for ${params.source.sourcePath}`); + } + if ( + !(await writeRecoveredSanitizedAuditArchive({ + sourceLabel: params.source.label, + root, + relativePath: sanitizedRelativePath, + allRecordsJsonl: prepared.sanitizedJsonl, + candidateRecordsJsonl: serializePreparedAuditRecords(candidateRecords), + previousCheckpoint, + warnings, + })) + ) { + return result(false); + } + // Checkpoint the unscrubbed append before hardening/scrubbing. A retry can + // then prove the sanitized tail was already written instead of duplicating it. + // A merge-intent count describes its exact raw snapshot. Keep an existing + // intent untouched when retry has no later rows; the final raw checkpoint replaces it. + if (previousCheckpoint?.phase !== "merge-intent" || candidateRecords.length > 0) { + if ( + !(await recordLegacyAuditRawCheckpoint({ + stateDir: params.stateDir, + rawPath: params.source.sourcePath, + rawRelativePath: claimRelativePath, + sanitizedRelativePath, + root, + snapshot, + phase: "merge-intent", + recordCount: prepared.records.length, + recordOrdinalBase: previousCheckpoint?.recordOrdinalBase ?? 0, + warnings, + })) + ) { + return result(false); + } + } + if ( + !(await secureAuditArchiveFile({ + root, + relativePath: sanitizedRelativePath, + label: `sanitized ${params.source.label}`, + warnings, + })) + ) { + return result(false); + } + if (missing.length > 0) { + changes.push( + `Recovered ${missing.length} later ${params.source.label} row(s) from ${params.source.sourcePath}${retentionNote}`, + ); + } + const scrubbedSnapshot = await scrubLegacyAuditRecoveryArchive({ + root, + relativePath: claimRelativePath, + expectedSnapshot: snapshot, + label: params.source.label, + warnings, + }); + if (!scrubbedSnapshot) { + return result(false); + } + const scrubbedRecords = prepareLegacyAuditRecords( + params.source, + scrubbedSnapshot.raw, + legacyAuditSourceGenerationKey(rawArchiveRelativePath), + ); + if (!scrubbedRecords.ok) { + warnings.push(...scrubbedRecords.warnings); + warnings.push( + `Retained uncheckpointed ${params.source.label} recovery archive; rerun openclaw doctor --fix`, + ); + return result(false); + } + if (scrubbedRecords.records.length !== 0) { + warnings.push( + `A legacy ${params.source.label} writer appended during recovery; rerun openclaw doctor --fix to import the retained rows`, + ); + return result(false); + } + const checkpointed = await recordLegacyAuditRawCheckpoint({ + stateDir: params.stateDir, + rawPath: params.source.sourcePath, + rawRelativePath: claimRelativePath, + sanitizedRelativePath, + root, + snapshot: scrubbedSnapshot, + phase: "raw", + recordCount: 0, + recordOrdinalBase: + (previousCheckpoint?.recordOrdinalBase ?? 0) + + Math.max(previousCheckpoint?.recordCount ?? 0, prepared.records.length), + warnings, + }); + if (checkpointed) { + await finalizeLegacyAuditRecoveryArchive({ root, relativePath: claimRelativePath }).catch( + (error: unknown) => { + warnings.push( + `Failed removing completed ${params.source.label} recovery journal: ${String(error)}`, + ); + }, + ); + } + return result(checkpointed); + } + if (!archivePaths) { + throw new Error(`Missing archive generation for ${params.source.sourcePath}`); + } + changes.push( + `Migrated ${params.source.label} -> shared SQLite state (${missing.length} new row(s)${retentionNote})`, + ); + const archived = await archiveLegacyAuditClaim({ + source: params.source, + claimRelativePath, + archivePaths, + snapshot, + sanitizedJsonl: prepared.sanitizedJsonl, + root, + changes, + warnings, + }); + claimFinalized = archived.moved; + if (!archived.moved || !archived.rawRelativePath) { + changes.pop(); + return result(false); + } + if (!archived.scrubbedSnapshot) { + return result(false); + } + const scrubbedRecords = prepareLegacyAuditRecords( + params.source, + archived.scrubbedSnapshot.raw, + legacyAuditSourceGenerationKey(archived.rawRelativePath), + ); + if (!scrubbedRecords.ok) { + warnings.push(...scrubbedRecords.warnings); + warnings.push( + `Retained uncheckpointed ${params.source.label} recovery archive; rerun openclaw doctor --fix`, + ); + return result(false); + } + if (scrubbedRecords.records.length !== 0) { + warnings.push( + `A legacy ${params.source.label} writer appended during migration; rerun openclaw doctor --fix to import the retained rows`, + ); + return result(false); + } + const rawPath = path.join(params.stateDir, archived.rawRelativePath); + const checkpointed = await recordLegacyAuditRawCheckpoint({ + stateDir: params.stateDir, + rawPath, + rawRelativePath: archived.rawRelativePath, + sanitizedRelativePath: archivePaths.sanitized, + root, + snapshot: archived.scrubbedSnapshot, + phase: "raw", + recordCount: 0, + recordOrdinalBase: prepared.records.length, + warnings, + }); + if (checkpointed) { + await finalizeLegacyAuditRecoveryArchive({ + root, + relativePath: archived.rawRelativePath, + }).catch((error: unknown) => { + warnings.push( + `Failed removing completed ${params.source.label} recovery journal: ${String(error)}`, + ); + }); + } + if ((await root.exists(sourceRelativePath)) && !params.recreatedSourceScheduled) { + warnings.push( + `An old writer recreated ${params.source.label} at ${params.source.logicalSourcePath}; rerun openclaw doctor --fix to import the retained rows`, + ); + } + return result(checkpointed); + } finally { + if (!claimFinalized && params.source.storage === "active" && archivePaths) { + await restoreOrPreserveLegacyAuditClaim({ + source: params.source, + claimRelativePath, + sourceRelativePath, + archivePaths, + root, + warnings, + }); + } + } +} + +export async function migrateLegacyAuditLogs(params: { + detected: LegacyAuditLogsDetection; + stateDir: string; +}): Promise { + const changes: string[] = []; + const warnings: string[] = []; + if (params.detected.sources.length === 0) { + return { changes, warnings }; + } + const env = { ...process.env, OPENCLAW_STATE_DIR: params.stateDir }; + let lock: Awaited>; + try { + // Exclusive state ownership excludes a predecessor Gateway and sibling doctor. + // Each source is also atomically claimed because old short-lived CLI processes + // can append config audit rows without participating in the Gateway lock. + lock = await acquireGatewayLock({ + allowInTests: true, + env, + pollIntervalMs: 25, + role: "sqlite-maintenance", + timeoutMs: 250, + }); + } catch (error) { + warnings.push( + `Skipped legacy audit migration because exclusive state ownership is unavailable: ${String(error)}`, + ); + return { changes, warnings }; + } + if (!lock) { + warnings.push( + "Skipped legacy audit migration because exclusive state ownership is unavailable", + ); + return { changes, warnings }; + } + try { + await withLegacyAuditMigrationLease(params.stateDir, async () => { + const blockedLogicalSources = new Set(); + for (const [index, source] of params.detected.sources.entries()) { + if (blockedLogicalSources.has(source.logicalSourcePath)) { + continue; + } + try { + const recreatedSourceScheduled = params.detected.sources + .slice(index + 1) + .some( + (candidate) => + candidate.storage === "active" && + candidate.logicalSourcePath === source.logicalSourcePath, + ); + const result = await migrateLegacyAuditLogSource({ + source, + stateDir: params.stateDir, + ...(recreatedSourceScheduled ? { recreatedSourceScheduled: true } : {}), + }); + changes.push(...result.changes); + warnings.push(...result.warnings); + if (!result.completed) { + // Generations encode append order. A later archive must not overtake + // an older source that still needs repair or durable checkpointing. + blockedLogicalSources.add(source.logicalSourcePath); + } + } catch (error) { + warnings.push(`Failed migrating ${source.label}: ${String(error)}`); + blockedLogicalSources.add(source.logicalSourcePath); + } + } + }); + } catch (error) { + warnings.push(`Skipped legacy audit migration because coordination failed: ${String(error)}`); + } finally { + await lock.release(); + } + return { changes, warnings }; +} diff --git a/src/infra/state-migrations.audit-logs.types.ts b/src/infra/state-migrations.audit-logs.types.ts new file mode 100644 index 00000000000..1dac8ee027f --- /dev/null +++ b/src/infra/state-migrations.audit-logs.types.ts @@ -0,0 +1,14 @@ +export type LegacyAuditLogSource = { + kind: "config" | "system-agent" | "crestodian"; + label: string; + sourcePath: string; + logicalSourcePath: string; + storage: "active" | "claim" | "raw-archive"; + sanitizedArchivePath?: string; + rawArchivePath?: string; +}; + +export type LegacyAuditLogsDetection = { + sources: LegacyAuditLogSource[]; + hasLegacy: boolean; +}; diff --git a/src/infra/state-migrations.audit-records.ts b/src/infra/state-migrations.audit-records.ts new file mode 100644 index 00000000000..9eb61cae95d --- /dev/null +++ b/src/infra/state-migrations.audit-records.ts @@ -0,0 +1,90 @@ +import { createHash } from "node:crypto"; +import { sanitizeConfigAuditRecord, type ConfigAuditRecord } from "../config/io.audit.js"; +import { redactSecrets } from "../logging/redact.js"; +import type { SystemAgentAuditEntry } from "../system-agent/audit.js"; +import type { LegacyAuditLogSource } from "./state-migrations.audit-logs.types.js"; + +export type PreparedAuditRecord = { + key: string; + value: ConfigAuditRecord | SystemAgentAuditEntry; + createdAt: number; +}; + +export function serializePreparedAuditRecords(records: readonly PreparedAuditRecord[]): string { + return records.length > 0 + ? `${records.map((record) => JSON.stringify(record.value)).join("\n")}\n` + : ""; +} + +function legacyAuditRecordCreatedAt( + source: LegacyAuditLogSource, + value: ConfigAuditRecord | SystemAgentAuditEntry, +): number { + const timestamp = + source.kind === "config" + ? (value as Partial).ts + : (value as Partial).timestamp; + if (typeof timestamp !== "string") { + return 0; + } + const parsed = Date.parse(timestamp); + return Number.isFinite(parsed) ? parsed : 0; +} + +type PreparedLegacyAuditRecords = + | { ok: false; warnings: string[] } + | { + ok: true; + records: PreparedAuditRecord[]; + sanitizedJsonl: string; + }; + +export function prepareLegacyAuditRecords( + source: LegacyAuditLogSource, + raw: string, + sourceGeneration: string, + sourceOrdinalBase = 0, +): PreparedLegacyAuditRecords { + const records: PreparedAuditRecord[] = []; + const warnings: string[] = []; + for (const [index, line] of raw.split(/\r?\n/u).entries()) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + let parsed: unknown; + try { + parsed = JSON.parse(trimmed) as unknown; + } catch (error) { + warnings.push( + `Failed reading ${source.label} record at ${source.sourcePath}:${index + 1}: ${String(error)}`, + ); + continue; + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + warnings.push( + `Skipped non-object ${source.label} record at ${source.sourcePath}:${index + 1}`, + ); + continue; + } + const value = + source.kind === "config" + ? sanitizeConfigAuditRecord(parsed as ConfigAuditRecord) + : (redactSecrets(parsed) as SystemAgentAuditEntry); + const digest = createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16); + const recordOrdinal = sourceOrdinalBase + records.length + 1; + records.push({ + key: `legacy:${source.kind}:${sourceGeneration}:${recordOrdinal}:${digest}`, + value, + createdAt: legacyAuditRecordCreatedAt(source, value), + }); + } + if (warnings.length > 0) { + return { ok: false, warnings }; + } + return { + ok: true, + records, + sanitizedJsonl: serializePreparedAuditRecords(records), + }; +} diff --git a/src/infra/state-migrations.audit-recovery-protocol.ts b/src/infra/state-migrations.audit-recovery-protocol.ts new file mode 100644 index 00000000000..afc61860900 --- /dev/null +++ b/src/infra/state-migrations.audit-recovery-protocol.ts @@ -0,0 +1,161 @@ +import { createHash } from "node:crypto"; +import type { LegacyAuditFileCheckpoint } from "./state-migrations.audit-checkpoints.js"; + +type AuditRecoveryRestoreJournal = { + schemaVersion: 6; + rawBase64: string; + scrubPatternBase64: string; + target: Pick; +}; + +export type ParsedAuditRecoveryRestoreJournal = { + sourceRaw: Buffer; + scrubPattern: Buffer; + target: AuditRecoveryRestoreJournal["target"]; + journalHash: string; +}; + +export type AuditRecoveryProgress = { + schemaVersion: 1; + journalHash: string; + direction: "restoring" | "scrubbing"; + committedBytes: number; + pendingEnd: number; + extentBytes: number; +}; + +const AUDIT_RECOVERY_SCRUB_PATTERN_BYTES = 32; + +export function serializeAuditRecoveryRestoreJournal(params: { + rawBytes: Buffer; + scrubPattern: Buffer; + target: AuditRecoveryRestoreJournal["target"]; +}): string { + const journal: AuditRecoveryRestoreJournal = { + schemaVersion: 6, + rawBase64: params.rawBytes.toString("base64"), + scrubPatternBase64: params.scrubPattern.toString("base64"), + target: params.target, + }; + return `${JSON.stringify(journal)}\n`; +} + +export function parseAuditRecoveryRestoreJournal(raw: string): ParsedAuditRecoveryRestoreJournal { + const parsed = JSON.parse(raw) as Partial; + if ( + parsed.schemaVersion !== 6 || + typeof parsed.rawBase64 !== "string" || + typeof parsed.scrubPatternBase64 !== "string" || + !parsed.target || + typeof parsed.target.dev !== "number" || + typeof parsed.target.ino !== "number" || + typeof parsed.target.size !== "number" + ) { + throw new Error("invalid legacy audit recovery restore journal"); + } + const scrubPattern = Buffer.from(parsed.scrubPatternBase64, "base64"); + if ( + scrubPattern.length !== AUDIT_RECOVERY_SCRUB_PATTERN_BYTES || + scrubPattern.some((byte) => byte !== 0x09 && byte !== 0x20) + ) { + throw new Error("invalid legacy audit recovery scrub pattern"); + } + const sourceRaw = Buffer.from(parsed.rawBase64, "base64"); + if (sourceRaw.length !== parsed.target.size) { + throw new Error("invalid legacy audit recovery source size"); + } + return { + sourceRaw, + scrubPattern, + target: parsed.target, + journalHash: createHash("sha256").update(raw).digest("hex"), + }; +} + +export function serializeAuditRecoveryProgress(progress: AuditRecoveryProgress): string { + return `${JSON.stringify(progress)}\n`; +} + +export function parseAuditRecoveryProgress( + raw: string, + journal: ParsedAuditRecoveryRestoreJournal, +): AuditRecoveryProgress { + const parsed = JSON.parse(raw) as Partial; + if ( + parsed.schemaVersion !== 1 || + parsed.journalHash !== journal.journalHash || + (parsed.direction !== "restoring" && parsed.direction !== "scrubbing") || + !Number.isSafeInteger(parsed.committedBytes) || + !Number.isSafeInteger(parsed.pendingEnd) || + !Number.isSafeInteger(parsed.extentBytes) || + parsed.committedBytes! < 0 || + parsed.pendingEnd! < parsed.committedBytes! || + parsed.extentBytes! < parsed.pendingEnd! || + parsed.extentBytes! > journal.target.size || + (parsed.direction === "scrubbing" && parsed.extentBytes !== journal.target.size) + ) { + throw new Error("invalid legacy audit recovery progress"); + } + return parsed as AuditRecoveryProgress; +} + +function auditRecoveryTransitionMatches( + current: Buffer, + previous: Buffer, + desired: Buffer, + start: number, + end: number, +): boolean { + let boundary = start; + while (boundary < end && current[boundary] === desired[boundary]) { + boundary += 1; + } + return current.subarray(boundary, end).equals(previous.subarray(boundary, end)); +} + +export function auditRecoveryStateMatchesJournal(params: { + current: Buffer; + original: Buffer; + scrubbed: Buffer; + progress: AuditRecoveryProgress; +}): boolean { + const { current, original, scrubbed, progress } = params; + if (current.length < original.length) { + return false; + } + if (progress.direction === "scrubbing") { + return ( + current + .subarray(0, progress.committedBytes) + .equals(scrubbed.subarray(0, progress.committedBytes)) && + auditRecoveryTransitionMatches( + current, + original, + scrubbed, + progress.committedBytes, + progress.pendingEnd, + ) && + current + .subarray(progress.pendingEnd, original.length) + .equals(original.subarray(progress.pendingEnd)) + ); + } + return ( + current + .subarray(0, progress.committedBytes) + .equals(original.subarray(0, progress.committedBytes)) && + auditRecoveryTransitionMatches( + current, + scrubbed, + original, + progress.committedBytes, + progress.pendingEnd, + ) && + current + .subarray(progress.pendingEnd, progress.extentBytes) + .equals(scrubbed.subarray(progress.pendingEnd, progress.extentBytes)) && + current + .subarray(progress.extentBytes, original.length) + .equals(original.subarray(progress.extentBytes)) + ); +} diff --git a/src/infra/state-migrations.audit-recovery.test.ts b/src/infra/state-migrations.audit-recovery.test.ts new file mode 100644 index 00000000000..c8316f394cf --- /dev/null +++ b/src/infra/state-migrations.audit-recovery.test.ts @@ -0,0 +1,542 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; +import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; +import { listSystemAgentAuditEntriesForTests } from "../system-agent/audit.test-support.js"; +import { withTempDir } from "../test-helpers/temp-dir.js"; +import { openLegacyAuditRawCheckpointStore } from "./state-migrations.audit-checkpoints.js"; +import { detectLegacyAuditLogs, migrateLegacyAuditLogs } from "./state-migrations.audit-logs.js"; + +const TEST_AUDIT_SCRUB_PATTERN = Buffer.from( + Array.from({ length: 32 }, (_, index) => (index % 2 === 0 ? 0x20 : 0x09)), +); + +function buildTestAuditScrubbedContent(length: number): Buffer { + const content = Buffer.allocUnsafe(length); + for (let offset = 0; offset < length; offset += TEST_AUDIT_SCRUB_PATTERN.length) { + TEST_AUDIT_SCRUB_PATTERN.copy( + content, + offset, + 0, + Math.min(TEST_AUDIT_SCRUB_PATTERN.length, length - offset), + ); + } + return content; +} + +async function buildTestAuditRestoreJournal( + rawPath: string, + sourceRaw: Buffer, + progress: { restoredBytes: number; scrubbedBytes: number } = { + restoredBytes: 0, + scrubbedBytes: 0, + }, +): Promise { + const stat = await fs.stat(rawPath); + const journal = `${JSON.stringify({ + schemaVersion: 6, + rawBase64: sourceRaw.toString("base64"), + scrubPatternBase64: TEST_AUDIT_SCRUB_PATTERN.toString("base64"), + target: { dev: stat.dev, ino: stat.ino, size: sourceRaw.length }, + })}\n`; + await fs.writeFile( + `${rawPath}.doctor-scrub-progress`, + `${JSON.stringify({ + schemaVersion: 1, + journalHash: createHash("sha256").update(journal).digest("hex"), + direction: progress.restoredBytes > 0 ? "restoring" : "scrubbing", + committedBytes: progress.restoredBytes > 0 ? progress.restoredBytes : progress.scrubbedBytes, + pendingEnd: progress.restoredBytes > 0 ? progress.restoredBytes : progress.scrubbedBytes, + extentBytes: progress.restoredBytes > 0 ? progress.scrubbedBytes : sourceRaw.length, + })}\n`, + ); + return journal; +} + +describe("legacy audit recovery byte handling", () => { + afterEach(() => { + resetPluginStateStoreForTests(); + }); + + it("blanks a zero-slack source within the fixed-size recovery inode", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-short-secret-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl"); + const record = { + ts: "2026-07-01T00:00:00.000Z", + source: "config-io", + event: "config.write", + argv: ["openclaw", "config", "set", "token", "x"], + execArgv: [], + }; + const original = `${JSON.stringify(record)}\n`; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, original); + + const result = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(result.warnings).toEqual([]); + const sanitized = await fs.readFile(`${sourcePath}.migrated`, "utf8"); + const recovery = await fs.readFile(`${sourcePath}.migrated.raw`, "utf8"); + expect(Buffer.byteLength(recovery)).toBe(Buffer.byteLength(original)); + expect(JSON.parse(sanitized.trim())).toMatchObject({ + argv: ["openclaw", "config", "set", "token", "***"], + }); + expect(recovery.trim()).toBe(""); + await expect( + fs.access(`${sourcePath}.migrated.raw.doctor-scrub-restore`), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); + + it("uses original byte offsets when decoded audit text contains replacement characters", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-invalid-utf8-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const original = Buffer.concat([ + Buffer.from( + '{"timestamp":"2026-07-03T00:00:00.000Z","operation":"gateway.restart","summary":"', + ), + Buffer.from([0x80]), + Buffer.from('"}'), + ]); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, original); + + const migrated = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(migrated.warnings).toEqual([]); + const blanked = await fs.readFile(rawPath); + expect(blanked).toHaveLength(original.length); + expect(blanked.every((byte) => byte === 0x20 || byte === 0x09)).toBe(true); + + await fs.appendFile( + rawPath, + `${JSON.stringify({ + timestamp: "2026-07-04T00:00:00.000Z", + operation: "gateway.restart", + summary: "later", + })}\n`, + ); + const recovered = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(recovered.warnings).toEqual([]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["�", "later"]); + }); + }); + + it("upgrades a legacy nonzero raw checkpoint to the blank append pad", async () => { + await withTempDir( + { prefix: "openclaw-audit-migration-checkpoint-upgrade-" }, + async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile( + sourcePath, + `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "original", + })}\n`, + ); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + const rawPath = `${sourcePath}.migrated.raw`; + const legacyRaw = Buffer.concat([ + Buffer.from( + '{"timestamp":"2026-07-03T00:00:00.000Z","operation":"gateway.restart","summary":"', + ), + Buffer.from([0x80]), + Buffer.from('"}\n'), + ]); + await fs.writeFile(rawPath, legacyRaw); + const legacyStat = await fs.stat(rawPath); + const checkpointStore = openLegacyAuditRawCheckpointStore(stateDir); + const checkpoint = checkpointStore.entries()[0]!; + checkpointStore.upsert(checkpoint.key, { + ...checkpoint.value, + dev: legacyStat.dev, + ino: legacyStat.ino, + mtimeMs: legacyStat.mtimeMs, + size: legacyStat.size, + contentHash: createHash("sha256").update(legacyRaw.toString("utf8")).digest("hex"), + recordCount: 1, + }); + + const detected = detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }); + expect(detected.hasLegacy).toBe(true); + const result = await migrateLegacyAuditLogs({ detected, stateDir }); + + expect(result.warnings).toEqual([]); + expect(openLegacyAuditRawCheckpointStore(stateDir).entries()[0]?.value.recordCount).toBe(0); + }, + ); + }); + + it("does not confuse an older prefix checkpoint with a later scrub generation", async () => { + await withTempDir( + { prefix: "openclaw-audit-migration-scrub-generation-" }, + async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const restorePath = `${rawPath}.doctor-scrub-restore`; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile( + sourcePath, + `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "original", + })}\n`, + ); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + await fs.appendFile( + rawPath, + `${JSON.stringify({ + timestamp: "2026-07-04T00:00:00.000Z", + operation: "gateway.restart", + summary: "later", + })}\n`, + ); + const interruptedRaw = await fs.readFile(rawPath); + await fs.writeFile( + restorePath, + await buildTestAuditRestoreJournal(rawPath, interruptedRaw, { + restoredBytes: 0, + scrubbedBytes: interruptedRaw.length, + }), + ); + await fs.writeFile(rawPath, buildTestAuditScrubbedContent(interruptedRaw.length)); + + const result = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(result.warnings).toEqual([]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["original", "later"]); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + }); + + it("does not replay a scrub journal over an equal-width space redaction", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-scrub-redacted-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const restorePath = `${rawPath}.doctor-scrub-restore`; + const originalContent = `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "secret archive value", + })}\n`; + const replacementContent = originalContent.replace("secret archive value", " ".repeat(20)); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(`${sourcePath}.migrated`, "{}\n"); + await fs.writeFile(rawPath, replacementContent); + await fs.writeFile( + restorePath, + await buildTestAuditRestoreJournal(rawPath, Buffer.from(originalContent, "utf8")), + ); + + const result = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(result.warnings.join("\n")).toContain("no longer matches its restore journal target"); + await expect(fs.readFile(rawPath, "utf8")).resolves.toBe(replacementContent); + await expect(fs.access(restorePath)).resolves.toBeUndefined(); + }); + }); + + it("resumes a one-byte scrub interruption using exact journal progress", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-scrub-one-byte-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const restorePath = `${rawPath}.doctor-scrub-restore`; + const originalContent = `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "original archive", + })}\n`; + const replacementBytes = Buffer.from(originalContent); + replacementBytes[0] = TEST_AUDIT_SCRUB_PATTERN[0]!; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(`${sourcePath}.migrated`, "{}\n"); + await fs.writeFile(rawPath, replacementBytes); + await fs.writeFile( + restorePath, + await buildTestAuditRestoreJournal(rawPath, Buffer.from(originalContent), { + restoredBytes: 0, + scrubbedBytes: 1, + }), + ); + + const result = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(result.warnings).toEqual([]); + expect((await fs.readFile(rawPath, "utf8")).trim()).toBe(""); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); + + it("resumes restoration after an interrupted rollback write", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-restore-restart-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const restorePath = `${rawPath}.doctor-scrub-restore`; + const originalContent = `${JSON.stringify({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "restore after restart", + })}\n`; + const originalBytes = Buffer.from(originalContent, "utf8"); + const interruptedRestore = buildTestAuditScrubbedContent(originalBytes.length); + originalBytes.subarray(0, Math.floor(originalBytes.length / 2)).copy(interruptedRestore); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(`${sourcePath}.migrated`, "{}\n"); + await fs.writeFile(rawPath, interruptedRestore); + await fs.writeFile( + restorePath, + await buildTestAuditRestoreJournal(rawPath, originalBytes, { + restoredBytes: Math.floor(originalBytes.length / 2), + scrubbedBytes: originalBytes.length, + }), + ); + + const result = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(result.warnings).toEqual([]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["restore after restart"]); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); + + it("retries raw archive recovery when sanitized archive hardening fails", async () => { + await withTempDir( + { prefix: "openclaw-audit-migration-recovery-permissions-" }, + async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const event = (summary: string) => ({ + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary, + }); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, `${JSON.stringify(event("before archive"))}\n`); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + await fs.appendFile( + `${sourcePath}.migrated.raw`, + `${JSON.stringify(event("later row"))}\n`, + ); + const probe = await fs.open(path.join(stateDir, "recovery-chmod-probe"), "w"); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + chmod(mode: number): Promise; + }; + await probe.close(); + const originalChmod = Reflect.get(fileHandlePrototype, "chmod") as ( + this: typeof fileHandlePrototype, + mode: number, + ) => Promise; + let chmodCalls = 0; + const chmodSpy = vi.spyOn(fileHandlePrototype, "chmod").mockImplementation(function ( + this: typeof fileHandlePrototype, + mode: number, + ) { + chmodCalls += 1; + if (chmodCalls === 2) { + return Promise.reject(new Error("simulated recovery chmod failure")); + } + return originalChmod.call(this, mode); + }); + + let failed: Awaited>; + try { + failed = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + } finally { + chmodSpy.mockRestore(); + } + + expect(failed.changes).toEqual([]); + expect(failed.warnings.join("\n")).toContain( + "Failed securing sanitized system-agent audit log", + ); + expect( + detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).sources, + ).toMatchObject([{ storage: "raw-archive" }]); + + const recovered = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + expect(recovered.warnings).toEqual([]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["before archive", "later row"]); + const sanitizedRows = (await fs.readFile(`${sourcePath}.migrated`, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { summary: string }); + expect(sanitizedRows.map((row) => row.summary)).toEqual(["before archive", "later row"]); + expect( + detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).sources, + ).toEqual([]); + }, + ); + }); + + it("completes a verified partial sanitized tail after an interrupted write", async () => { + await withTempDir( + { prefix: "openclaw-audit-migration-partial-sanitized-tail-" }, + async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const sanitizedPath = `${sourcePath}.migrated`; + const rawPath = `${sanitizedPath}.raw`; + const event = (day: string, summary: string) => ({ + timestamp: `2026-07-${day}T00:00:00.000Z`, + operation: "gateway.restart", + summary, + }); + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, `${JSON.stringify(event("01", "before archive"))}\n`); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + const firstLater = event("02", "first later row"); + const secondLater = event("03", "second later row"); + await fs.appendFile( + rawPath, + `${JSON.stringify(firstLater)}\n${JSON.stringify(secondLater)}\n`, + ); + // Simulate a stopped sanitized write: the durable checkpoint prefix plus + // one complete candidate row is a byte-for-byte prefix of the desired file. + await fs.appendFile(sanitizedPath, `${JSON.stringify(firstLater)}\n`); + + const recovered = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(recovered.warnings).toEqual([]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["before archive", "first later row", "second later row"]); + const sanitizedRows = (await fs.readFile(sanitizedPath, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { summary: string }); + expect(sanitizedRows.map((row) => row.summary)).toEqual([ + "before archive", + "first later row", + "second later row", + ]); + }, + ); + }); + + it("preserves identical appends and blocks checkpointless whitespace ambiguity", async () => { + await withTempDir({ prefix: "openclaw-audit-migration-duplicate-tail-" }, async (stateDir) => { + const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl"); + const rawPath = `${sourcePath}.migrated.raw`; + const event = { + timestamp: "2026-07-03T00:00:00.000Z", + operation: "gateway.restart", + summary: "Repeated operation", + }; + await fs.mkdir(path.dirname(sourcePath), { recursive: true }); + await fs.writeFile(sourcePath, `${JSON.stringify(event)}\n\n${JSON.stringify(event)}\n`); + await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + await fs.appendFile(rawPath, `${JSON.stringify(event)}\n`); + + const recovered = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + + expect(recovered.warnings).toEqual([]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }).map((entry) => entry.value.summary), + ).toEqual(["Repeated operation", "Repeated operation", "Repeated operation"]); + const sanitizedRows = (await fs.readFile(`${sourcePath}.migrated`, "utf8")) + .trim() + .split("\n"); + expect(sanitizedRows).toHaveLength(3); + + runOpenClawStateWriteTransaction( + (database) => { + database.db + .prepare("DELETE FROM diagnostic_events WHERE scope = ?") + .run("migration.legacy-audit-raw"); + }, + { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }, + ); + await fs.appendFile(rawPath, `${JSON.stringify(event)}\n`); + const ambiguous = await migrateLegacyAuditLogs({ + detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }), + stateDir, + }); + expect(ambiguous.changes).toEqual([]); + expect(ambiguous.warnings).toEqual([ + expect.stringContaining("checkpointless raw archive begins with ambiguous whitespace"), + ]); + expect( + listSystemAgentAuditEntriesForTests({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toHaveLength(3); + expect((await fs.readFile(`${sourcePath}.migrated`, "utf8")).trim().split("\n")).toHaveLength( + 3, + ); + }); + }); +}); diff --git a/src/infra/state-migrations.audit-recovery.ts b/src/infra/state-migrations.audit-recovery.ts new file mode 100644 index 00000000000..68ed2a45912 --- /dev/null +++ b/src/infra/state-migrations.audit-recovery.ts @@ -0,0 +1,710 @@ +import { createHash, randomBytes } from "node:crypto"; +import path from "node:path"; +import { root as createFsSafeRoot } from "./fs-safe.js"; +import { syncDirectoryBestEffort } from "./sqlite-snapshot.js"; +import { + legacyAuditRawCheckpointKey, + legacyAuditRawCheckpointsMatch, + legacyAuditSourceGenerationKey, + openLegacyAuditRawCheckpointStore, + type LegacyAuditFileCheckpoint, + type LegacyAuditRawCheckpoint, +} from "./state-migrations.audit-checkpoints.js"; +import { + auditRecoveryStateMatchesJournal, + parseAuditRecoveryProgress, + parseAuditRecoveryRestoreJournal, + serializeAuditRecoveryProgress, + serializeAuditRecoveryRestoreJournal, + type AuditRecoveryProgress, + type ParsedAuditRecoveryRestoreJournal, +} from "./state-migrations.audit-recovery-protocol.js"; + +export type AuditMigrationRoot = Awaited>; +export type LegacyAuditSourceSnapshot = LegacyAuditFileCheckpoint & { + raw: string; + rawBytes: Buffer; +}; + +const AUDIT_RECOVERY_RESTORE_SUFFIX = ".doctor-scrub-restore"; +const AUDIT_RECOVERY_STAGING_SUFFIX = ".doctor-scrub-staging"; +const AUDIT_RECOVERY_PROGRESS_SUFFIX = ".doctor-scrub-progress"; +const AUDIT_RECOVERY_SCRUB_PATTERN_BYTES = 32; + +function auditRecoverySiblingPath(relativePath: string, suffix: string): string { + return `${relativePath}${suffix}`; +} + +function auditRecoveryJournalTargetsSnapshot( + snapshot: LegacyAuditSourceSnapshot, + journal: Pick, +): boolean { + return ( + snapshot.dev === journal.target.dev && + snapshot.ino === journal.target.ino && + snapshot.rawBytes.length >= journal.target.size + ); +} + +function auditRecoveryCheckpointPrefixMatches( + snapshot: LegacyAuditSourceSnapshot, + checkpoint: LegacyAuditRawCheckpoint, +): boolean { + if (snapshot.rawBytes.length < checkpoint.size) { + return false; + } + return ( + createHash("sha256").update(snapshot.rawBytes.subarray(0, checkpoint.size)).digest("hex") === + checkpoint.contentHash + ); +} + +async function syncAuditRecoveryDirectory( + root: AuditMigrationRoot, + relativePath: string, +): Promise { + await syncDirectoryBestEffort(path.join(root.rootReal, path.dirname(relativePath))); +} + +export async function readLegacyAuditSourceSnapshot( + root: AuditMigrationRoot, + relativePath: string, +): Promise { + const opened = await root.open(relativePath); + try { + const before = await opened.handle.stat(); + if (!before.isFile()) { + throw new Error("legacy audit source is not a regular file"); + } + const rawBytes = await opened.handle.readFile(); + const after = await opened.handle.stat(); + const beforeCheckpoint = { + dev: before.dev, + ino: before.ino, + mtimeMs: before.mtimeMs, + size: before.size, + }; + const afterCheckpoint = { + dev: after.dev, + ino: after.ino, + mtimeMs: after.mtimeMs, + size: after.size, + }; + if (!legacyAuditRawCheckpointsMatch(beforeCheckpoint, afterCheckpoint)) { + throw new Error("legacy audit source changed while Doctor was reading it"); + } + return { ...afterCheckpoint, raw: rawBytes.toString("utf8"), rawBytes }; + } finally { + await opened.handle.close(); + } +} + +export async function readLegacyAuditSourcePrefixSnapshotForBackup( + root: AuditMigrationRoot, + relativePath: string, +): Promise { + const opened = await root.open(relativePath); + try { + const before = await opened.handle.stat(); + if (!before.isFile()) { + throw new Error("legacy audit source is not a regular file"); + } + const rawBytes = Buffer.allocUnsafe(before.size); + let offset = 0; + while (offset < rawBytes.length) { + const length = Math.min(64 * 1024, rawBytes.length - offset); + const { bytesRead } = await opened.handle.read(rawBytes, offset, length, offset); + if (bytesRead === 0) { + throw new Error("legacy audit source was truncated while backup was reading it"); + } + offset += bytesRead; + } + const after = await opened.handle.stat(); + if (before.dev !== after.dev || before.ino !== after.ino || after.size < before.size) { + throw new Error("legacy audit source changed other than by append during backup"); + } + const checkpoint = { + dev: before.dev, + ino: before.ino, + mtimeMs: before.mtimeMs, + size: before.size, + }; + return { ...checkpoint, raw: rawBytes.toString("utf8"), rawBytes }; + } finally { + await opened.handle.close(); + } +} + +export async function readLegacyAuditRecoverySourceForBackup( + root: AuditMigrationRoot, + relativePath: string, +): Promise { + const current = await readLegacyAuditSourcePrefixSnapshotForBackup(root, relativePath); + const restoreRelativePath = auditRecoverySiblingPath(relativePath, AUDIT_RECOVERY_RESTORE_SUFFIX); + if (!(await root.exists(restoreRelativePath))) { + return current; + } + const restoreSnapshot = await readLegacyAuditSourceSnapshot(root, restoreRelativePath); + const journal = parseAuditRecoveryRestoreJournal(restoreSnapshot.raw); + const progress = await readAuditRecoveryProgress({ root, relativePath, journal }); + const scrubbedContent = buildScrubbedAuditRecoveryContent( + journal.sourceRaw, + journal.scrubPattern, + ); + if ( + !auditRecoveryJournalTargetsSnapshot(current, journal) || + !auditRecoveryStateMatchesJournal({ + current: current.rawBytes, + original: journal.sourceRaw, + scrubbed: scrubbedContent, + progress, + }) + ) { + return current; + } + const rawBytes = Buffer.concat([ + journal.sourceRaw, + current.rawBytes.subarray(journal.sourceRaw.length), + ]); + return { ...current, raw: rawBytes.toString("utf8"), rawBytes }; +} + +function createAuditRecoveryScrubPattern(): Buffer { + const pattern = randomBytes(AUDIT_RECOVERY_SCRUB_PATTERN_BYTES); + for (let index = 0; index < pattern.length; index += 1) { + pattern[index] = (pattern[index]! & 1) === 0 ? 0x20 : 0x09; + } + for (let offset = 0; offset < pattern.length; offset += 8) { + const block = pattern.subarray(offset, offset + 8); + if (block.every((byte) => byte === 0x20)) { + pattern[offset + 7] = 0x09; + } else if (block.every((byte) => byte === 0x09)) { + pattern[offset + 7] = 0x20; + } + } + return pattern; +} + +function buildScrubbedAuditRecoveryContent(rawBytes: Buffer, scrubPattern: Buffer): Buffer { + if (rawBytes.length === 0) { + return Buffer.alloc(0); + } + // The readable sanitized sibling owns migrated history. This same-inode file + // is only an append landing pad for predecessor writers, so blank the complete + // fixed-size prefix and checkpoint it with zero records. Leading whitespace is + // valid before any late JSONL row and preserves an open O_APPEND offset. + const scrubbed = Buffer.allocUnsafe(rawBytes.length); + for (let offset = 0; offset < scrubbed.length; offset += scrubPattern.length) { + scrubPattern.copy(scrubbed, offset, 0, Math.min(scrubPattern.length, scrubbed.length - offset)); + } + return scrubbed; +} + +async function writeAuditRecoveryRange( + handle: Awaited>["handle"], + content: Buffer, + position: number, +): Promise { + let offset = 0; + while (offset < content.byteLength) { + const { bytesWritten } = await handle.write( + content, + offset, + content.byteLength - offset, + position + offset, + ); + if (bytesWritten === 0) { + throw new Error("zero-byte write while updating legacy recovery archive"); + } + offset += bytesWritten; + } +} + +const AUDIT_RECOVERY_WRITE_CHUNK_BYTES = 64 * 1024; + +async function writeAuditRecoveryProgress(params: { + root: AuditMigrationRoot; + relativePath: string; + progress: AuditRecoveryProgress; +}): Promise { + const progressRelativePath = auditRecoverySiblingPath( + params.relativePath, + AUDIT_RECOVERY_PROGRESS_SUFFIX, + ); + await params.root.write(progressRelativePath, serializeAuditRecoveryProgress(params.progress), { + mkdir: false, + mode: 0o600, + }); + const opened = await params.root.open(progressRelativePath); + try { + await opened.handle.chmod(0o600); + await opened.handle.sync(); + } finally { + await opened.handle.close(); + } + await syncAuditRecoveryDirectory(params.root, params.relativePath); +} + +async function readAuditRecoveryProgress(params: { + root: AuditMigrationRoot; + relativePath: string; + journal: ReturnType; +}): Promise { + const progressRelativePath = auditRecoverySiblingPath( + params.relativePath, + AUDIT_RECOVERY_PROGRESS_SUFFIX, + ); + if (!(await params.root.exists(progressRelativePath))) { + return { + schemaVersion: 1, + journalHash: params.journal.journalHash, + direction: "scrubbing", + committedBytes: 0, + pendingEnd: 0, + extentBytes: params.journal.target.size, + }; + } + const snapshot = await readLegacyAuditSourceSnapshot(params.root, progressRelativePath); + return parseAuditRecoveryProgress(snapshot.raw, params.journal); +} + +async function advanceAuditRecoveryWrite(params: { + root: AuditMigrationRoot; + relativePath: string; + progress: AuditRecoveryProgress; + desiredContent: Buffer; + handle: Awaited>["handle"]; +}): Promise { + let progress = params.progress; + if (progress.pendingEnd > progress.committedBytes) { + await writeAuditRecoveryRange( + params.handle, + params.desiredContent.subarray(progress.committedBytes, progress.pendingEnd), + progress.committedBytes, + ); + await params.handle.sync(); + progress = { ...progress, committedBytes: progress.pendingEnd }; + await writeAuditRecoveryProgress({ ...params, progress }); + } + while (progress.committedBytes < progress.extentBytes) { + const end = Math.min( + progress.committedBytes + AUDIT_RECOVERY_WRITE_CHUNK_BYTES, + progress.extentBytes, + ); + // Commit intent before target bytes. A crash can leave any prefix of this + // range changed; pendingEnd lets recovery finish it without guessing. + progress = { ...progress, pendingEnd: end }; + await writeAuditRecoveryProgress({ ...params, progress }); + await writeAuditRecoveryRange( + params.handle, + params.desiredContent.subarray(progress.committedBytes, end), + progress.committedBytes, + ); + await params.handle.sync(); + progress = { ...progress, committedBytes: end }; + await writeAuditRecoveryProgress({ ...params, progress }); + } + return progress; +} + +async function reconcileAuditRecoveryPendingWrite(params: { + root: AuditMigrationRoot; + relativePath: string; + progress: AuditRecoveryProgress; + desiredContent: Buffer; + handle: Awaited>["handle"]; +}): Promise { + if (params.progress.pendingEnd === params.progress.committedBytes) { + return params.progress; + } + await writeAuditRecoveryRange( + params.handle, + params.desiredContent.subarray(params.progress.committedBytes, params.progress.pendingEnd), + params.progress.committedBytes, + ); + await params.handle.sync(); + const progress = { ...params.progress, committedBytes: params.progress.pendingEnd }; + await writeAuditRecoveryProgress({ ...params, progress }); + return progress; +} + +async function stageAuditRecoveryRestore(params: { + root: AuditMigrationRoot; + relativePath: string; + snapshot: LegacyAuditSourceSnapshot; + scrubPattern: Buffer; +}): Promise { + const restoreRelativePath = auditRecoverySiblingPath( + params.relativePath, + AUDIT_RECOVERY_RESTORE_SUFFIX, + ); + const stagingRelativePath = auditRecoverySiblingPath( + params.relativePath, + AUDIT_RECOVERY_STAGING_SUFFIX, + ); + await params.root.remove(stagingRelativePath).catch(() => undefined); + const journalRaw = serializeAuditRecoveryRestoreJournal({ + rawBytes: params.snapshot.rawBytes, + scrubPattern: params.scrubPattern, + target: { + dev: params.snapshot.dev, + ino: params.snapshot.ino, + size: params.snapshot.size, + }, + }); + await params.root.create(stagingRelativePath, journalRaw, { mode: 0o600 }); + const staged = await params.root.open(stagingRelativePath); + try { + await staged.handle.chmod(0o600); + await staged.handle.sync(); + } finally { + await staged.handle.close(); + } + await params.root.move(stagingRelativePath, restoreRelativePath); + await syncAuditRecoveryDirectory(params.root, params.relativePath); + const journal = parseAuditRecoveryRestoreJournal(journalRaw); + const progress: AuditRecoveryProgress = { + schemaVersion: 1, + journalHash: journal.journalHash, + direction: "scrubbing", + committedBytes: 0, + pendingEnd: 0, + extentBytes: journal.target.size, + }; + await writeAuditRecoveryProgress({ + root: params.root, + relativePath: params.relativePath, + progress, + }); + return progress; +} + +export async function restoreInterruptedAuditRecoveryArchive(params: { + root: AuditMigrationRoot; + relativePath: string; + label: string; + warnings: string[]; +}): Promise { + const restoreRelativePath = auditRecoverySiblingPath( + params.relativePath, + AUDIT_RECOVERY_RESTORE_SUFFIX, + ); + const stagingRelativePath = auditRecoverySiblingPath( + params.relativePath, + AUDIT_RECOVERY_STAGING_SUFFIX, + ); + const progressRelativePath = auditRecoverySiblingPath( + params.relativePath, + AUDIT_RECOVERY_PROGRESS_SUFFIX, + ); + if (!(await params.root.exists(restoreRelativePath))) { + await params.root.remove(stagingRelativePath).catch(() => undefined); + await params.root.remove(progressRelativePath).catch(() => undefined); + return true; + } + try { + const currentSnapshot = await readLegacyAuditSourceSnapshot(params.root, params.relativePath); + const restoreSnapshot = await readLegacyAuditSourceSnapshot(params.root, restoreRelativePath); + const journal = parseAuditRecoveryRestoreJournal(restoreSnapshot.raw); + let progress = await readAuditRecoveryProgress({ + root: params.root, + relativePath: params.relativePath, + journal, + }); + const scrubbedContent = buildScrubbedAuditRecoveryContent( + journal.sourceRaw, + journal.scrubPattern, + ); + let completedCheckpoint: LegacyAuditRawCheckpoint | undefined; + try { + completedCheckpoint = findPreviousLegacyAuditRawCheckpoint( + params.root.rootReal, + params.relativePath, + ); + } catch { + completedCheckpoint = undefined; + } + if ( + completedCheckpoint && + completedCheckpoint.phase === "raw" && + completedCheckpoint.recordCount === 0 && + completedCheckpoint.size === journal.sourceRaw.length && + auditRecoveryCheckpointPrefixMatches(currentSnapshot, completedCheckpoint) + ) { + // Checkpoint commit won the crash race; the restore journal is stale and + // must not roll the already-checkpointed sanitized inode backward. + await params.root.remove(progressRelativePath).catch(() => undefined); + await params.root.remove(stagingRelativePath).catch(() => undefined); + await params.root.remove(restoreRelativePath); + await syncAuditRecoveryDirectory(params.root, params.relativePath); + return true; + } + const writable = await params.root.openWritable(params.relativePath, { + mode: 0o600, + writeMode: "update", + }); + try { + const verification = await readLegacyAuditSourceSnapshot(params.root, params.relativePath); + if ( + writable.stat.dev !== verification.dev || + writable.stat.ino !== verification.ino || + !auditRecoveryJournalTargetsSnapshot(verification, journal) || + !auditRecoveryStateMatchesJournal({ + current: verification.rawBytes, + original: journal.sourceRaw, + scrubbed: scrubbedContent, + progress, + }) + ) { + throw new Error("legacy recovery archive no longer matches its restore journal target"); + } + progress = await reconcileAuditRecoveryPendingWrite({ + root: params.root, + relativePath: params.relativePath, + progress, + desiredContent: progress.direction === "scrubbing" ? scrubbedContent : journal.sourceRaw, + handle: writable.handle, + }); + if (progress.direction === "scrubbing") { + progress = { + schemaVersion: 1, + journalHash: journal.journalHash, + direction: "restoring", + committedBytes: 0, + pendingEnd: 0, + extentBytes: progress.committedBytes, + }; + await writeAuditRecoveryProgress({ + root: params.root, + relativePath: params.relativePath, + progress, + }); + } + await advanceAuditRecoveryWrite({ + root: params.root, + relativePath: params.relativePath, + progress, + desiredContent: journal.sourceRaw, + handle: writable.handle, + }); + await writable.handle.chmod(0o600); + await writable.handle.sync(); + } finally { + await writable.handle.close().catch(() => undefined); + } + await params.root.remove(progressRelativePath).catch(() => undefined); + await params.root.remove(stagingRelativePath).catch(() => undefined); + await params.root.remove(restoreRelativePath); + await syncAuditRecoveryDirectory(params.root, params.relativePath); + return true; + } catch (error) { + params.warnings.push( + `Failed restoring interrupted ${params.label} legacy recovery archive: ${String(error)}`, + ); + return false; + } +} + +export async function finalizeLegacyAuditRecoveryArchive(params: { + root: AuditMigrationRoot; + relativePath: string; +}): Promise { + await params.root + .remove(auditRecoverySiblingPath(params.relativePath, AUDIT_RECOVERY_PROGRESS_SUFFIX)) + .catch(() => undefined); + await params.root + .remove(auditRecoverySiblingPath(params.relativePath, AUDIT_RECOVERY_STAGING_SUFFIX)) + .catch(() => undefined); + await params.root.remove( + auditRecoverySiblingPath(params.relativePath, AUDIT_RECOVERY_RESTORE_SUFFIX), + ); + await syncAuditRecoveryDirectory(params.root, params.relativePath); +} + +export async function scrubLegacyAuditRecoveryArchive(params: { + root: AuditMigrationRoot; + relativePath: string; + expectedSnapshot: LegacyAuditSourceSnapshot; + label: string; + warnings: string[]; +}): Promise { + const scrubPattern = createAuditRecoveryScrubPattern(); + const scrubbedContent = buildScrubbedAuditRecoveryContent( + params.expectedSnapshot.rawBytes, + scrubPattern, + ); + let progress: AuditRecoveryProgress; + try { + progress = await stageAuditRecoveryRestore({ + root: params.root, + relativePath: params.relativePath, + snapshot: params.expectedSnapshot, + scrubPattern, + }); + } catch (error) { + params.warnings.push( + `Failed staging ${params.label} legacy recovery restore journal: ${String(error)}`, + ); + return undefined; + } + let writable: Awaited>; + try { + writable = await params.root.openWritable(params.relativePath, { + mode: 0o600, + writeMode: "update", + }); + } catch (error) { + params.warnings.push( + `Failed scrubbing ${params.label} legacy recovery archive: ${String(error)}`, + ); + return undefined; + } + try { + if (!legacyAuditRawCheckpointsMatch(params.expectedSnapshot, writable.stat)) { + params.warnings.push( + `Skipped scrubbing changed ${params.label} legacy recovery archive; rerun openclaw doctor --fix`, + ); + return undefined; + } + await advanceAuditRecoveryWrite({ + root: params.root, + relativePath: params.relativePath, + progress, + desiredContent: scrubbedContent, + handle: writable.handle, + }); + await writable.handle.chmod(0o600); + await writable.handle.sync(); + } catch (error) { + await writable.handle.close().catch(() => undefined); + const recoveryWarnings: string[] = []; + const restored = await restoreInterruptedAuditRecoveryArchive({ + root: params.root, + relativePath: params.relativePath, + label: params.label, + warnings: recoveryWarnings, + }); + if (restored) { + params.warnings.push( + `Failed scrubbing ${params.label} legacy recovery archive; restored it for Doctor retry: ${String(error)}`, + ); + } else { + params.warnings.push(...recoveryWarnings); + params.warnings.push( + `Failed scrubbing ${params.label} legacy recovery archive; left its progress journal for Doctor retry: ${String(error)}`, + ); + } + return undefined; + } finally { + await writable.handle.close().catch(() => undefined); + } + let scrubbedSnapshot: LegacyAuditSourceSnapshot; + try { + scrubbedSnapshot = await readLegacyAuditSourceSnapshot(params.root, params.relativePath); + } catch (error) { + params.warnings.push( + `Changed ${params.label} legacy recovery archive during scrub verification; rerun openclaw doctor --fix: ${String(error)}`, + ); + return undefined; + } + const scrubbedPrefix = scrubbedSnapshot.rawBytes.subarray(0, scrubbedContent.length); + if (!scrubbedPrefix.equals(scrubbedContent)) { + params.warnings.push( + `Failed verifying scrubbed ${params.label} legacy recovery archive; rerun openclaw doctor --fix`, + ); + return undefined; + } + return scrubbedSnapshot; +} + +export async function recordLegacyAuditRawCheckpoint(params: { + stateDir: string; + rawPath: string; + rawRelativePath: string; + sanitizedRelativePath: string; + root: AuditMigrationRoot; + snapshot: LegacyAuditSourceSnapshot; + phase: "merge-intent" | "raw"; + recordCount: number; + recordOrdinalBase: number; + warnings: string[]; +}): Promise { + try { + const sanitizedSnapshot = await readLegacyAuditSourceSnapshot( + params.root, + params.sanitizedRelativePath, + ); + const opened = await params.root.open(params.rawRelativePath); + let checkpoint: LegacyAuditRawCheckpoint; + try { + const stat = await opened.handle.stat(); + checkpoint = { + dev: stat.dev, + ino: stat.ino, + mtimeMs: stat.mtimeMs, + size: stat.size, + phase: params.phase, + generationKey: legacyAuditSourceGenerationKey(params.rawRelativePath), + recordCount: params.recordCount, + recordOrdinalBase: params.recordOrdinalBase, + contentHash: createHash("sha256").update(params.snapshot.rawBytes).digest("hex"), + sanitizedContentHash: createHash("sha256").update(sanitizedSnapshot.rawBytes).digest("hex"), + sanitizedSize: sanitizedSnapshot.rawBytes.length, + }; + } finally { + await opened.handle.close(); + } + if (!legacyAuditRawCheckpointsMatch(checkpoint, params.snapshot)) { + params.warnings.push( + `Retained changed legacy audit backup ${params.rawPath}; rerun openclaw doctor --fix to import its later rows`, + ); + return false; + } + openLegacyAuditRawCheckpointStore(params.stateDir).upsert( + legacyAuditRawCheckpointKey(checkpoint), + checkpoint, + ); + return true; + } catch (error) { + params.warnings.push( + `Failed recording legacy audit backup checkpoint for ${params.rawPath}: ${String(error)}`, + ); + return false; + } +} + +export function findPreviousLegacyAuditRawCheckpoint( + stateDir: string, + rawRelativePath: string, +): LegacyAuditRawCheckpoint | undefined { + const generationKey = legacyAuditSourceGenerationKey(rawRelativePath); + return openLegacyAuditRawCheckpointStore(stateDir) + .entries() + .toReversed() + .find((entry) => entry.value.generationKey === generationKey)?.value; +} + +export function recordsAfterLegacyAuditRawCheckpoint(params: { + checkpoint: LegacyAuditRawCheckpoint; + snapshot: LegacyAuditSourceSnapshot; + records: readonly T[]; +}): readonly T[] | undefined { + const rawBytes = params.snapshot.rawBytes; + if (rawBytes.length < params.checkpoint.size) { + return undefined; + } + const prefixHash = createHash("sha256") + .update(rawBytes.subarray(0, params.checkpoint.size)) + .digest("hex"); + const legacyUtf8PrefixHash = createHash("sha256") + .update(rawBytes.subarray(0, params.checkpoint.size).toString("utf8")) + .digest("hex"); + if ( + (prefixHash !== params.checkpoint.contentHash && + legacyUtf8PrefixHash !== params.checkpoint.contentHash) || + params.records.length < params.checkpoint.recordCount + ) { + return undefined; + } + return params.records.slice(params.checkpoint.recordCount); +} diff --git a/src/infra/state-migrations.audit-sanitized.ts b/src/infra/state-migrations.audit-sanitized.ts new file mode 100644 index 00000000000..0ab995de0d3 --- /dev/null +++ b/src/infra/state-migrations.audit-sanitized.ts @@ -0,0 +1,72 @@ +import { createHash } from "node:crypto"; +import type { LegacyAuditRawCheckpoint } from "./state-migrations.audit-checkpoints.js"; +import { + readLegacyAuditSourceSnapshot, + type AuditMigrationRoot, +} from "./state-migrations.audit-recovery.js"; + +export async function writeRecoveredSanitizedAuditArchive(params: { + sourceLabel: string; + root: AuditMigrationRoot; + relativePath: string; + allRecordsJsonl: string; + candidateRecordsJsonl: string; + previousCheckpoint: LegacyAuditRawCheckpoint | undefined; + warnings: string[]; +}): Promise { + const current = (await params.root.exists(params.relativePath)) + ? await readLegacyAuditSourceSnapshot(params.root, params.relativePath) + : undefined; + let desired: Buffer; + if (params.previousCheckpoint) { + if (!current || current.rawBytes.length < params.previousCheckpoint.sanitizedSize) { + params.warnings.push( + `Skipped ${params.sourceLabel} recovery because its sanitized archive is missing or truncated`, + ); + return false; + } + const checkpointedPrefix = current.rawBytes.subarray( + 0, + params.previousCheckpoint.sanitizedSize, + ); + if ( + createHash("sha256").update(checkpointedPrefix).digest("hex") !== + params.previousCheckpoint.sanitizedContentHash + ) { + params.warnings.push( + `Skipped ${params.sourceLabel} recovery because its sanitized archive changed after checkpoint`, + ); + return false; + } + // The caller already removed checkpointed raw records. For merge-intent, + // this sanitized prefix also contains that previously materialized batch. + desired = Buffer.concat([ + checkpointedPrefix, + Buffer.from(params.candidateRecordsJsonl, "utf8"), + ]); + if (current.rawBytes.equals(desired)) { + return true; + } + const currentIsVerifiedDesiredPrefix = desired + .subarray(0, current.rawBytes.length) + .equals(current.rawBytes); + if ( + current.rawBytes.length !== params.previousCheckpoint.sanitizedSize && + !currentIsVerifiedDesiredPrefix + ) { + params.warnings.push( + `Skipped ${params.sourceLabel} recovery because its sanitized archive has an uncheckpointed tail`, + ); + return false; + } + } else { + // A checkpointless unscrubbed raw archive is authoritative. Replacing from + // the complete sanitized projection is idempotent before checkpoint commit. + desired = Buffer.from(params.allRecordsJsonl, "utf8"); + if (current?.rawBytes.equals(desired)) { + return true; + } + } + await params.root.write(params.relativePath, desired, { mkdir: false, mode: 0o600 }); + return true; +} diff --git a/src/infra/state-migrations.doctor.ts b/src/infra/state-migrations.doctor.ts index 22a84774161..fc31680fabb 100644 --- a/src/infra/state-migrations.doctor.ts +++ b/src/infra/state-migrations.doctor.ts @@ -11,6 +11,8 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { createPluginStateKeyedStore, + getPluginStateCapacity as resolvePluginStateCapacity, + importPluginStateEntriesForDoctor, type OpenKeyedStoreOptions, } from "../plugin-state/plugin-state-store.js"; import { @@ -27,6 +29,7 @@ import { repairOpenClawStateDatabaseSchema, type OpenClawStateDatabaseSchemaMigration, } from "../state/openclaw-state-db.js"; +import { acquireGatewayLock } from "./gateway-lock.js"; import { detectLegacyAcpReplayLedger, migrateLegacyAcpReplayLedger, @@ -35,6 +38,7 @@ import { detectLegacyApnsRegistrations, migrateLegacyApnsRegistrations, } from "./state-migrations.apns.js"; +import { detectLegacyAuditLogs, migrateLegacyAuditLogs } from "./state-migrations.audit-logs.js"; import { detectLegacyChannelPairingState, migrateLegacyChannelPairingState, @@ -164,6 +168,9 @@ function describeStateSchemaMigration(migration: OpenClawStateDatabaseSchemaMigr let autoMigrateChecked = false; +const PLUGIN_DOCTOR_MIGRATION_LOCK_TIMEOUT_MS = 250; +const PLUGIN_DOCTOR_MIGRATION_LOCK_POLL_INTERVAL_MS = 25; + export function resetAutoMigrateLegacyStateForTest(): void { autoMigrateChecked = false; resetAutoMigrateLegacyTaskStateSidecarsForTest(); @@ -206,6 +213,7 @@ async function collectPluginDoctorStateMigrationPlans(params: { env: NodeJS.ProcessEnv; stateDir: string; oauthDir: string; + includeDoctorOnly?: boolean; warnings?: string[]; }): Promise { const plans: DetectedPluginDoctorStateMigrationPlan[] = []; @@ -214,6 +222,9 @@ async function collectPluginDoctorStateMigrationPlans(params: { config, env: params.env, })) { + if (entry.migration.doctorOnly === true && params.includeDoctorOnly !== true) { + continue; + } let detected: PluginDoctorStateMigrationDetection | null; try { detected = await entry.migration.detectLegacyState({ @@ -243,6 +254,15 @@ function createPluginDoctorStateMigrationContext( env: NodeJS.ProcessEnv, ): PluginDoctorStateMigrationContext { return { + getPluginStateCapacity() { + return resolvePluginStateCapacity(pluginId, env); + }, + importPluginStateEntries( + options: OpenKeyedStoreOptions, + entries: readonly { key: string; value: unknown; createdAt: number }[], + ) { + importPluginStateEntriesForDoctor(pluginId, { ...options, env: options.env ?? env }, entries); + }, openPluginStateKeyedStore(options: OpenKeyedStoreOptions) { return createPluginStateKeyedStore(pluginId, { ...options, @@ -406,6 +426,10 @@ export async function detectLegacyStateMigrations(params: { stateDir, doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, }); + const auditLogs = detectLegacyAuditLogs({ + stateDir, + doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, + }); const acpReplayLedger = detectLegacyAcpReplayLedger({ stateDir, doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, @@ -522,6 +546,7 @@ export async function detectLegacyStateMigrations(params: { env, stateDir, oauthDir, + includeDoctorOnly: params.doctorOnlyStateMigrations === true, warnings: pluginPlanWarnings, }); @@ -593,6 +618,9 @@ export async function detectLegacyStateMigrations(params: { if (commitments.hasLegacy) { preview.push("- Commitments: legacy JSON file → shared SQLite state"); } + for (const source of auditLogs.sources) { + preview.push(`- ${source.label}: legacy JSONL file → shared SQLite state`); + } if (acpReplayLedger.hasLegacy) { preview.push("- ACP replay ledger: legacy JSON file → shared SQLite state"); } @@ -634,6 +662,7 @@ export async function detectLegacyStateMigrations(params: { } return { + doctorOnlyStateMigrations: params.doctorOnlyStateMigrations === true, targetAgentId, targetMainKey, targetScope, @@ -707,6 +736,7 @@ export async function detectLegacyStateMigrations(params: { }, tuiLastSessions, commitments, + auditLogs, acpReplayLedger, managedOutgoingImages, apns, @@ -737,6 +767,7 @@ async function runPluginDoctorStateMigrationPlans(params: { env: params.env, stateDir: params.detected.stateDir, oauthDir: params.detected.oauthDir, + includeDoctorOnly: params.detected.doctorOnlyStateMigrations, warnings, }); const hasDetectorFailure = warnings.length > 0; @@ -746,21 +777,80 @@ async function runPluginDoctorStateMigrationPlans(params: { refreshedPlans.length > 0 || hasDetectorFailure ? refreshedPlans : (params.detected.pluginPlans?.plans ?? []); - for (const plan of plans) { - try { - const result = await plan.migration.migrateLegacyState({ - config: params.config, - env: params.env, - stateDir: params.detected.stateDir, - oauthDir: params.detected.oauthDir, - context: createPluginDoctorStateMigrationContext(plan.pluginId, params.env), - }); - changes.push(...result.changes); - warnings.push(...result.warnings); - notices.push(...(result.notices ?? [])); - } catch (err) { - warnings.push(`Failed migrating ${plan.migration.label}: ${String(err)}`); + const migrated = await migratePluginDoctorStatePlans({ + plans, + config: params.config, + env: params.env, + stateDir: params.detected.stateDir, + oauthDir: params.detected.oauthDir, + }); + changes.push(...migrated.changes); + warnings.push(...migrated.warnings); + notices.push(...(migrated.notices ?? [])); + return notices.length > 0 ? { changes, warnings, notices } : { changes, warnings }; +} + +async function migratePluginDoctorStatePlans(params: { + plans: readonly DetectedPluginDoctorStateMigrationPlan[]; + config: OpenClawConfig; + env: NodeJS.ProcessEnv; + stateDir: string; + oauthDir: string; +}): Promise { + const changes: string[] = []; + const warnings: string[] = []; + const notices: string[] = []; + if (params.plans.length === 0) { + return { changes, warnings }; + } + + let lock: Awaited>; + try { + lock = await acquireGatewayLock({ + allowInTests: true, + env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir }, + pollIntervalMs: PLUGIN_DOCTOR_MIGRATION_LOCK_POLL_INTERVAL_MS, + role: "sqlite-maintenance", + timeoutMs: PLUGIN_DOCTOR_MIGRATION_LOCK_TIMEOUT_MS, + }); + } catch (error) { + return { + changes, + warnings: [ + `Skipped plugin doctor state migrations because exclusive state ownership is unavailable: ${String(error)}`, + ], + }; + } + if (!lock) { + return { + changes, + warnings: [ + "Skipped plugin doctor state migrations because exclusive state ownership is unavailable", + ], + }; + } + + try { + // Plugin migrations may claim retired files after verified import. Keep the + // predecessor Gateway excluded for the full read, import, and archive window. + for (const plan of params.plans) { + try { + const result = await plan.migration.migrateLegacyState({ + config: params.config, + env: params.env, + stateDir: params.stateDir, + oauthDir: params.oauthDir, + context: createPluginDoctorStateMigrationContext(plan.pluginId, params.env), + }); + changes.push(...result.changes); + warnings.push(...result.warnings); + notices.push(...(result.notices ?? [])); + } catch (err) { + warnings.push(`Failed migrating ${plan.migration.label}: ${String(err)}`); + } } + } finally { + await lock.release(); } return notices.length > 0 ? { changes, warnings, notices } : { changes, warnings }; } @@ -807,22 +897,16 @@ export async function autoMigrateLegacyPluginDoctorState(params: { oauthDir, warnings, }); - for (const plan of plans) { - try { - const result = await plan.migration.migrateLegacyState({ - config: params.config, - env, - stateDir, - oauthDir, - context: createPluginDoctorStateMigrationContext(plan.pluginId, env), - }); - changes.push(...result.changes); - warnings.push(...result.warnings); - notices.push(...(result.notices ?? [])); - } catch (err) { - warnings.push(`Failed migrating ${plan.migration.label}: ${String(err)}`); - } - } + const migrated = await migratePluginDoctorStatePlans({ + plans, + config: params.config, + env, + stateDir, + oauthDir, + }); + changes.push(...migrated.changes); + warnings.push(...migrated.warnings); + notices.push(...(migrated.notices ?? [])); return { migrated: stateDirResult.migrated || stateSchema.changes.length > 0 || plans.length > 0, skipped: false, @@ -902,6 +986,10 @@ export async function runLegacyStateMigrations(params: { detected: detected.commitments, stateDir: detected.stateDir, }); + const auditLogs = await migrateLegacyAuditLogs({ + detected: detected.auditLogs, + stateDir: detected.stateDir, + }); const acpReplayLedger = await migrateLegacyAcpReplayLedger({ detected: detected.acpReplayLedger, stateDir: detected.stateDir, @@ -980,6 +1068,7 @@ export async function runLegacyStateMigrations(params: { updateCheck, tuiLastSessions, commitments, + auditLogs, acpReplayLedger, managedOutgoingImages, apns, @@ -1006,6 +1095,7 @@ export async function runLegacyStateMigrations(params: { ...currentConversationBindings.changes, ...tuiLastSessions.changes, ...commitments.changes, + ...auditLogs.changes, ...acpReplayLedger.changes, ...managedOutgoingImages.changes, ...apns.changes, @@ -1039,6 +1129,7 @@ export async function runLegacyStateMigrations(params: { ...currentConversationBindings.warnings, ...tuiLastSessions.warnings, ...commitments.warnings, + ...auditLogs.warnings, ...acpReplayLedger.warnings, ...managedOutgoingImages.warnings, ...apns.warnings, diff --git a/src/infra/state-migrations.test.ts b/src/infra/state-migrations.test.ts index 0ea28ea9ec3..e1995dfac46 100644 --- a/src/infra/state-migrations.test.ts +++ b/src/infra/state-migrations.test.ts @@ -16,6 +16,7 @@ import { OPENCLAW_STATE_SCHEMA_VERSION, } from "../state/openclaw-state-db.js"; import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; +import { acquireGatewayLock } from "./gateway-lock.js"; import { executeSqliteQuerySync, executeSqliteQueryTakeFirstSync, @@ -48,6 +49,7 @@ const pluginDoctorStateMigrationEntries = vi.hoisted( migration: { id: string; label: string; + doctorOnly?: boolean; detectLegacyState: (params: { config: OpenClawConfig; env: NodeJS.ProcessEnv; @@ -546,6 +548,54 @@ describe("state migrations", () => { expect(result.changes).toContain("Migrated fixture environment"); }); + it("runs doctor-only plugin file imports only during explicit Doctor repair", async () => { + const root = await createTempDir(); + const stateDir = path.join(root, ".openclaw"); + const env = createEnv(stateDir); + const cfg = createConfig(); + const detectLegacyState = vi.fn(() => ({ preview: ["doctor-only plugin state"] })); + const migrateLegacyState = vi.fn(() => ({ + changes: ["doctor-only plugin state migrated"], + warnings: [], + })); + pluginDoctorStateMigrationEntries.entries = [ + { + pluginId: "memory-core", + migration: { + id: "memory-core-doctor-only-test", + label: "Memory Core doctor-only test migration", + doctorOnly: true, + detectLegacyState, + migrateLegacyState, + }, + }, + ]; + + const automatic = await autoMigrateLegacyPluginDoctorState({ + config: cfg, + env, + homedir: () => root, + }); + expect(automatic.changes).not.toContain("doctor-only plugin state migrated"); + expect(detectLegacyState).not.toHaveBeenCalled(); + expect(migrateLegacyState).not.toHaveBeenCalled(); + + const detected = await detectLegacyStateMigrations({ + cfg, + env, + homedir: () => root, + doctorOnlyStateMigrations: true, + }); + expect(detected.pluginPlans).toMatchObject({ hasLegacy: true }); + expect(detected.preview).toContain("doctor-only plugin state"); + + const repaired = await runLegacyStateMigrations({ detected, config: cfg, env }); + expect(repaired.warnings).toStrictEqual([]); + expect(repaired.changes).toContain("doctor-only plugin state migrated"); + expect(detectLegacyState).toHaveBeenCalledTimes(2); + expect(migrateLegacyState).toHaveBeenCalledOnce(); + }); + it("detects legacy sessions, agent files, channel auth, and pairing state", () => { expect(detectionCase.targetAgentId).toBe("worker-1"); expect(detectionCase.targetMainKey).toBe("desk"); @@ -2011,6 +2061,55 @@ describe("state migrations", () => { expect(migrateLegacyState).toHaveBeenCalledOnce(); }); + it("requires exclusive state ownership before plugin doctor migrations", async () => { + const root = await createTempDir(); + const stateDir = path.join(root, ".openclaw"); + const env = createEnv(stateDir); + const cfg = createConfig(); + openOpenClawStateDatabase({ env }); + closeOpenClawStateDatabaseForTest(); + const migrateLegacyState = vi.fn(() => ({ + changes: ["plugin state migrated"], + warnings: [], + })); + pluginDoctorStateMigrationEntries.entries = [ + { + pluginId: "memory-core", + migration: { + id: "memory-core-lock-test", + label: "Memory Core lock test migration", + detectLegacyState: () => ({ preview: ["plugin state"] }), + migrateLegacyState, + }, + }, + ]; + const gatewayLock = await acquireGatewayLock({ + allowInTests: true, + env, + pollIntervalMs: 10, + port: 18_791, + timeoutMs: 100, + }); + if (!gatewayLock) { + throw new Error("expected test Gateway lock"); + } + + let result: Awaited>; + try { + result = await autoMigrateLegacyPluginDoctorState({ + config: cfg, + env, + homedir: () => root, + }); + } finally { + await gatewayLock.release(); + } + + expect(result.changes).not.toContain("plugin state migrated"); + expect(result.warnings.join("\n")).toContain("exclusive state ownership is unavailable"); + expect(migrateLegacyState).not.toHaveBeenCalled(); + }); + it("skips stale plugin doctor plans when refresh detection fails", async () => { const root = await createTempDir(); const stateDir = path.join(root, ".openclaw"); diff --git a/src/infra/state-migrations.types.ts b/src/infra/state-migrations.types.ts index bbe853f514f..5b3e3dcca3f 100644 --- a/src/infra/state-migrations.types.ts +++ b/src/infra/state-migrations.types.ts @@ -1,6 +1,7 @@ import type { ChannelLegacyStateMigrationPlan } from "../channels/plugins/types.core.js"; import type { SessionScope } from "../config/sessions/types.js"; import type { PluginDoctorStateMigration } from "../plugins/doctor-contract-registry.js"; +import type { LegacyAuditLogsDetection } from "./state-migrations.audit-logs.types.js"; import type { LegacyChannelPairingStateDetection } from "./state-migrations.channel-pairing.js"; import type { LegacyMcpOAuthDetection } from "./state-migrations.mcp-oauth.types.js"; import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js"; @@ -18,6 +19,7 @@ export type SessionStoreAliasPlan = { }; export type LegacyStateDetection = { + doctorOnlyStateMigrations?: boolean; targetAgentId: string; targetMainKey: string; targetScope?: SessionScope; @@ -103,6 +105,7 @@ export type LegacyStateDetection = { sourcePath: string; hasLegacy: boolean; }; + auditLogs: LegacyAuditLogsDetection; acpReplayLedger: { sourcePath: string; hasLegacy: boolean; diff --git a/src/memory-host-sdk/event-export.ts b/src/memory-host-sdk/event-export.ts new file mode 100644 index 00000000000..40c28fa742f --- /dev/null +++ b/src/memory-host-sdk/event-export.ts @@ -0,0 +1,20 @@ +import type { PersistedMemoryHostEvent } from "./event-store.js"; + +export const MAX_MEMORY_HOST_PUBLIC_EXPORT_BYTES = 1024 * 1024; + +export function serializeMemoryHostEventExport( + storedEvents: readonly PersistedMemoryHostEvent[], +): string { + const lines: string[] = []; + let sizeBytes = 0; + for (const entry of storedEvents.toReversed()) { + const line = JSON.stringify(entry.value.event); + const lineBytes = Buffer.byteLength(line, "utf8") + 1; + if (sizeBytes + lineBytes > MAX_MEMORY_HOST_PUBLIC_EXPORT_BYTES) { + break; + } + lines.push(line); + sizeBytes += lineBytes; + } + return lines.toReversed().join("\n") + "\n"; +} diff --git a/src/memory-host-sdk/event-store.ts b/src/memory-host-sdk/event-store.ts new file mode 100644 index 00000000000..96dc7c1c944 --- /dev/null +++ b/src/memory-host-sdk/event-store.ts @@ -0,0 +1,359 @@ +import { createHash } from "node:crypto"; +import { resolveWorkspaceStateIdentity } from "../agents/workspace-state-store.js"; +import { + pluginStateEntriesInKeyRange, + registerPluginStateSyncSequencedJournalEntry, +} from "../plugin-state/plugin-state-store.js"; +import type { MemoryHostEventRecord } from "./event-types.js"; + +const MEMORY_HOST_EVENTS_PLUGIN_ID = "memory-core"; +const MEMORY_HOST_EVENTS_NAMESPACE = "memory-host.events"; +const MEMORY_HOST_EVENT_CURSORS_NAMESPACE = "memory-host.event-cursors"; +// Event rotation retains deep diagnostics without consuming memory-core's +// plugin-wide 50,000-row budget or starving sibling state namespaces. +const MAX_MEMORY_HOST_EVENTS = 10_000; +const MAX_MEMORY_HOST_EVENT_CURSORS = 1_000; +const MAX_MEMORY_HOST_EVENT_JSON_BYTES = 8 * 1024; +const MAX_MEMORY_HOST_EVENT_ITEMS = 10; +const MAX_MEMORY_HOST_EVENT_TEXT_BYTES = 2 * 1024; +const MAX_MEMORY_HOST_EVENT_PATH_BYTES = 256; +const WORKSPACE_HASH_BYTES = 24; + +type StoredMemoryHostEvent = { + kind: "event"; + event: MemoryHostEventRecord; + recordedAt: number; + sequence: number; +}; + +let maxMemoryHostEventsForTests: number | undefined; + +export type PersistedMemoryHostEvent = { + key: string; + value: StoredMemoryHostEvent; + createdAt: number; +}; + +function normalizeMemoryHostWorkspaceKey(workspaceDir: string): string { + // Workspace aliases must share one event/cursor namespace. Otherwise two + // configured paths to the same workspace can publish conflicting exports. + const resolved = resolveWorkspaceStateIdentity(workspaceDir).workspacePath.replace(/\\/g, "/"); + return process.platform === "win32" ? resolved.toLowerCase() : resolved; +} + +function memoryHostWorkspacePrefix(workspaceDir: string): string { + return createHash("sha256") + .update(normalizeMemoryHostWorkspaceKey(workspaceDir)) + .digest("hex") + .slice(0, WORKSPACE_HASH_BYTES); +} + +function eventKeyPrefix(workspaceDir: string): string { + return `${memoryHostWorkspacePrefix(workspaceDir)}:event:`; +} + +function eventKeyRangeEnd(workspaceDir: string): string { + return `${memoryHostWorkspacePrefix(workspaceDir)}:event;`; +} + +function memoryHostEventStorageKey(workspaceDir: string, sequence: number): string { + if (!Number.isSafeInteger(sequence)) { + throw new Error("Memory host event sequence must be a safe integer"); + } + return `${eventKeyPrefix(workspaceDir)}1:${sequence.toString().padStart(16, "0")}`; +} + +function cursorKey(workspaceDir: string): string { + return `${memoryHostWorkspacePrefix(workspaceDir)}:cursor`; +} + +function truncateUtf8(value: string, maxBytes: number): { value: string; truncated: boolean } { + if (Buffer.byteLength(value, "utf8") <= maxBytes) { + return { value, truncated: false }; + } + let low = 0; + let high = value.length; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if (Buffer.byteLength(value.slice(0, middle), "utf8") <= maxBytes - 3) { + low = middle; + } else { + high = middle - 1; + } + } + const end = low > 0 && /[\uD800-\uDBFF]/u.test(value.charAt(low - 1)) ? low - 1 : low; + return { value: `${value.slice(0, end)}…`, truncated: true }; +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +/** Validate and bound one diagnostic event before storing it in plugin state. */ +export function normalizeMemoryHostEventRecordForStorage( + value: unknown, +): MemoryHostEventRecord | null { + if (!isRecord(value) || typeof value.type !== "string" || typeof value.timestamp !== "string") { + return null; + } + const timestamp = truncateUtf8(value.timestamp, 128); + let truncated = timestamp.truncated || value.storageTruncated === true; + + if (value.type === "memory.recall.recorded" || value.type === "memory.recall.skipped") { + if ( + typeof value.query !== "string" || + !Array.isArray(value.results) || + (value.type === "memory.recall.recorded" + ? !isFiniteNumber(value.resultCount) + : !isFiniteNumber(value.skippedResultCount)) + ) { + return null; + } + if ( + value.type === "memory.recall.skipped" && + (value.reason !== "non-short-term-memory-path" || + !isFiniteNumber(value.eligibleResultCount) || + !isFiniteNumber(value.skippedResultCount)) + ) { + return null; + } + const query = truncateUtf8(value.query, MAX_MEMORY_HOST_EVENT_TEXT_BYTES); + truncated ||= query.truncated || value.results.length > MAX_MEMORY_HOST_EVENT_ITEMS; + const results: Array<{ + path: string; + startLine: number; + endLine: number; + score: number; + reason?: "non-short-term-memory-path"; + }> = []; + for (const result of value.results.slice(0, MAX_MEMORY_HOST_EVENT_ITEMS)) { + if ( + !isRecord(result) || + typeof result.path !== "string" || + !isFiniteNumber(result.startLine) || + !isFiniteNumber(result.endLine) || + !isFiniteNumber(result.score) || + (value.type === "memory.recall.skipped" && result.reason !== "non-short-term-memory-path") + ) { + return null; + } + const resultPath = truncateUtf8(result.path, MAX_MEMORY_HOST_EVENT_PATH_BYTES); + truncated ||= resultPath.truncated; + results.push({ + path: resultPath.value, + startLine: result.startLine, + endLine: result.endLine, + score: result.score, + ...(value.type === "memory.recall.skipped" + ? { reason: "non-short-term-memory-path" as const } + : {}), + }); + } + const normalized = + value.type === "memory.recall.recorded" + ? { + type: "memory.recall.recorded" as const, + timestamp: timestamp.value, + query: query.value, + resultCount: value.resultCount as number, + results: results.map((result) => ({ + path: result.path, + startLine: result.startLine, + endLine: result.endLine, + score: result.score, + })), + ...(truncated ? { storageTruncated: true as const } : {}), + } + : { + type: "memory.recall.skipped" as const, + timestamp: timestamp.value, + query: query.value, + reason: "non-short-term-memory-path" as const, + eligibleResultCount: value.eligibleResultCount as number, + skippedResultCount: value.skippedResultCount as number, + results: results.map((result) => ({ + path: result.path, + startLine: result.startLine, + endLine: result.endLine, + score: result.score, + reason: "non-short-term-memory-path" as const, + })), + ...(truncated ? { storageTruncated: true as const } : {}), + }; + return Buffer.byteLength(JSON.stringify(normalized), "utf8") <= MAX_MEMORY_HOST_EVENT_JSON_BYTES + ? normalized + : { ...normalized, results: [], storageTruncated: true as const }; + } + + if (value.type === "memory.promotion.applied") { + if ( + typeof value.memoryPath !== "string" || + !isFiniteNumber(value.applied) || + !Array.isArray(value.candidates) + ) { + return null; + } + const memoryPath = truncateUtf8(value.memoryPath, MAX_MEMORY_HOST_EVENT_PATH_BYTES); + truncated ||= memoryPath.truncated || value.candidates.length > MAX_MEMORY_HOST_EVENT_ITEMS; + const candidates: Array<{ + key: string; + path: string; + startLine: number; + endLine: number; + score: number; + recallCount: number; + }> = []; + for (const candidate of value.candidates.slice(0, MAX_MEMORY_HOST_EVENT_ITEMS)) { + if ( + !isRecord(candidate) || + typeof candidate.key !== "string" || + typeof candidate.path !== "string" || + !isFiniteNumber(candidate.startLine) || + !isFiniteNumber(candidate.endLine) || + !isFiniteNumber(candidate.score) || + !isFiniteNumber(candidate.recallCount) + ) { + return null; + } + const key = truncateUtf8(candidate.key, MAX_MEMORY_HOST_EVENT_PATH_BYTES); + const candidatePath = truncateUtf8(candidate.path, MAX_MEMORY_HOST_EVENT_PATH_BYTES); + truncated ||= key.truncated || candidatePath.truncated; + candidates.push({ + key: key.value, + path: candidatePath.value, + startLine: candidate.startLine, + endLine: candidate.endLine, + score: candidate.score, + recallCount: candidate.recallCount, + }); + } + const normalized = { + type: "memory.promotion.applied" as const, + timestamp: timestamp.value, + memoryPath: memoryPath.value, + applied: value.applied, + candidates, + ...(truncated ? { storageTruncated: true as const } : {}), + }; + return Buffer.byteLength(JSON.stringify(normalized), "utf8") <= MAX_MEMORY_HOST_EVENT_JSON_BYTES + ? normalized + : { ...normalized, candidates: [], storageTruncated: true as const }; + } + + if (value.type === "memory.dream.completed") { + if ( + (value.phase !== "light" && value.phase !== "deep" && value.phase !== "rem") || + (value.outcome !== undefined && + value.outcome !== "completed" && + value.outcome !== "failed") || + (value.error !== undefined && typeof value.error !== "string") || + (value.inlinePath !== undefined && typeof value.inlinePath !== "string") || + (value.reportPath !== undefined && typeof value.reportPath !== "string") || + !isFiniteNumber(value.lineCount) || + (value.storageMode !== "inline" && + value.storageMode !== "separate" && + value.storageMode !== "both") + ) { + return null; + } + const error = value.error + ? truncateUtf8(value.error, MAX_MEMORY_HOST_EVENT_TEXT_BYTES) + : undefined; + const inlinePath = value.inlinePath + ? truncateUtf8(value.inlinePath, MAX_MEMORY_HOST_EVENT_PATH_BYTES) + : undefined; + const reportPath = value.reportPath + ? truncateUtf8(value.reportPath, MAX_MEMORY_HOST_EVENT_PATH_BYTES) + : undefined; + truncated ||= Boolean(error?.truncated || inlinePath?.truncated || reportPath?.truncated); + return { + type: value.type, + timestamp: timestamp.value, + phase: value.phase, + ...(value.outcome ? { outcome: value.outcome } : {}), + ...(error ? { error: error.value } : {}), + ...(inlinePath ? { inlinePath: inlinePath.value } : {}), + ...(reportPath ? { reportPath: reportPath.value } : {}), + lineCount: value.lineCount, + storageMode: value.storageMode, + ...(truncated ? { storageTruncated: true } : {}), + }; + } + + return null; +} + +export function registerMemoryHostEvent(params: { + workspaceDir: string; + event: MemoryHostEventRecord; + env?: NodeJS.ProcessEnv; +}): void { + const event = normalizeMemoryHostEventRecordForStorage(params.event); + if (!event) { + throw new TypeError("Memory host event is invalid"); + } + const initialSequence = Math.max( + 0, + listStoredMemoryHostEvents({ + workspaceDir: params.workspaceDir, + limit: 1, + ...(params.env ? { env: params.env } : {}), + }).at(-1)?.value.sequence ?? 0, + ); + const recordedAt = Date.now(); + registerPluginStateSyncSequencedJournalEntry({ + pluginId: MEMORY_HOST_EVENTS_PLUGIN_ID, + cursorOptions: { + namespace: MEMORY_HOST_EVENT_CURSORS_NAMESPACE, + maxEntries: MAX_MEMORY_HOST_EVENT_CURSORS, + ...(params.env ? { env: params.env } : {}), + }, + cursorKey: cursorKey(params.workspaceDir), + journalOptions: { + namespace: MEMORY_HOST_EVENTS_NAMESPACE, + maxEntries: maxMemoryHostEventsForTests ?? MAX_MEMORY_HOST_EVENTS, + ...(params.env ? { env: params.env } : {}), + }, + initialSequence, + journalKey: (sequence) => memoryHostEventStorageKey(params.workspaceDir, sequence), + journalValue: (sequence) => ({ kind: "event", event, recordedAt, sequence }), + }); +} + +export function listStoredMemoryHostEvents(params: { + workspaceDir: string; + limit?: number; + env?: NodeJS.ProcessEnv; +}): PersistedMemoryHostEvent[] { + const limit = Number.isFinite(params.limit) + ? Math.max( + 1, + Math.min( + maxMemoryHostEventsForTests ?? MAX_MEMORY_HOST_EVENTS, + Math.floor(params.limit as number), + ), + ) + : (maxMemoryHostEventsForTests ?? MAX_MEMORY_HOST_EVENTS); + const entries = pluginStateEntriesInKeyRange({ + pluginId: MEMORY_HOST_EVENTS_PLUGIN_ID, + namespace: MEMORY_HOST_EVENTS_NAMESPACE, + keyStartInclusive: eventKeyPrefix(params.workspaceDir), + keyEndExclusive: eventKeyRangeEnd(params.workspaceDir), + limit, + order: "desc", + ...(params.env ? { env: params.env } : {}), + }).flatMap((entry): PersistedMemoryHostEvent[] => { + const value = entry.value as StoredMemoryHostEvent; + return value.kind === "event" ? [{ ...entry, value }] : []; + }); + return entries.toReversed(); +} + +/** Test-only retention override; production keeps a 10,000-event namespace budget. */ +export function setMaxMemoryHostEventsForTests(maxEntries?: number): void { + maxMemoryHostEventsForTests = maxEntries; +} diff --git a/src/memory-host-sdk/event-types.ts b/src/memory-host-sdk/event-types.ts new file mode 100644 index 00000000000..f2bba538094 --- /dev/null +++ b/src/memory-host-sdk/event-types.ts @@ -0,0 +1,80 @@ +import type { MemoryDreamingPhaseName } from "./dreaming.js"; + +type MemoryHostEventStorageMetadata = { + /** True when diagnostic detail was bounded before persistence. Aggregate counts stay exact. */ + storageTruncated?: true; +}; + +/** Event emitted when a recall query records the selected memory snippets. */ +export type MemoryHostRecallRecordedEvent = MemoryHostEventStorageMetadata & { + type: "memory.recall.recorded"; + timestamp: string; + query: string; + resultCount: number; + results: Array<{ + path: string; + startLine: number; + endLine: number; + score: number; + }>; +}; + +/** Event emitted when recall hits are visible but excluded from short-term promotion. */ +export type MemoryHostRecallSkippedEvent = MemoryHostEventStorageMetadata & { + type: "memory.recall.skipped"; + timestamp: string; + query: string; + reason: "non-short-term-memory-path"; + eligibleResultCount: number; + skippedResultCount: number; + results: Array<{ + path: string; + startLine: number; + endLine: number; + score: number; + reason: "non-short-term-memory-path"; + }>; +}; + +/** Event emitted when deep-dream candidates are promoted into durable memory. */ +export type MemoryHostPromotionAppliedEvent = MemoryHostEventStorageMetadata & { + type: "memory.promotion.applied"; + timestamp: string; + memoryPath: string; + applied: number; + candidates: Array<{ + key: string; + path: string; + startLine: number; + endLine: number; + score: number; + recallCount: number; + }>; +}; + +/** Normalized outcome for a dreaming phase run. */ +export type MemoryDreamOutcome = "completed" | "failed"; + +/** Event emitted after a dreaming phase writes inline memory and/or reports. */ +export type MemoryHostDreamCompletedEvent = MemoryHostEventStorageMetadata & { + type: "memory.dream.completed"; + timestamp: string; + phase: MemoryDreamingPhaseName; + /** Missing on older event logs; readers should treat absent as "completed". */ + outcome?: MemoryDreamOutcome; + /** Error detail when outcome is "failed". */ + error?: string; + inlinePath?: string; + reportPath?: string; + lineCount: number; + storageMode: "inline" | "separate" | "both"; +}; + +/** Durable memory host events consumed by status and public-artifact readers. */ +export type MemoryHostEvent = + | MemoryHostRecallRecordedEvent + | MemoryHostPromotionAppliedEvent + | MemoryHostDreamCompletedEvent; + +/** Full event record schema, including opt-in diagnostic variants. */ +export type MemoryHostEventRecord = MemoryHostEvent | MemoryHostRecallSkippedEvent; diff --git a/src/memory-host-sdk/events.ts b/src/memory-host-sdk/events.ts index eac122e5557..06ae73d4f44 100644 --- a/src/memory-host-sdk/events.ts +++ b/src/memory-host-sdk/events.ts @@ -1,150 +1,52 @@ -// Memory host event helpers append and read memory host event logs. -import fs from "node:fs/promises"; +// Memory host event helpers append and read persisted memory host events. import path from "node:path"; -import { appendRegularFile } from "../infra/fs-safe.js"; -import type { MemoryDreamingPhaseName } from "./dreaming.js"; +import { + listStoredMemoryHostEvents, + normalizeMemoryHostEventRecordForStorage, + registerMemoryHostEvent, +} from "./event-store.js"; +import type { MemoryHostEvent, MemoryHostEventRecord } from "./event-types.js"; -/** Workspace-relative JSONL audit log for memory recall, promotion, and dream events. */ -export const MEMORY_HOST_EVENT_LOG_RELATIVE_PATH = path.join("memory", ".dreams", "events.jsonl"); - -/** Event emitted when a recall query records the selected memory snippets. */ -export type MemoryHostRecallRecordedEvent = { - type: "memory.recall.recorded"; - timestamp: string; - query: string; - resultCount: number; - results: Array<{ - path: string; - startLine: number; - endLine: number; - score: number; - }>; -}; - -/** Event emitted when recall hits are visible but excluded from short-term promotion. */ -export type MemoryHostRecallSkippedEvent = { - type: "memory.recall.skipped"; - timestamp: string; - query: string; - reason: "non-short-term-memory-path"; - eligibleResultCount: number; - skippedResultCount: number; - results: Array<{ - path: string; - startLine: number; - endLine: number; - score: number; - reason: "non-short-term-memory-path"; - }>; -}; - -/** Event emitted when deep-dream candidates are promoted into durable memory. */ -export type MemoryHostPromotionAppliedEvent = { - type: "memory.promotion.applied"; - timestamp: string; - memoryPath: string; - applied: number; - candidates: Array<{ - key: string; - path: string; - startLine: number; - endLine: number; - score: number; - recallCount: number; - }>; -}; +export type { + MemoryDreamOutcome, + MemoryHostDreamCompletedEvent, + MemoryHostEvent, + MemoryHostEventRecord, + MemoryHostPromotionAppliedEvent, + MemoryHostRecallRecordedEvent, + MemoryHostRecallSkippedEvent, +} from "./event-types.js"; -/** Normalized outcome for a dreaming phase run. */ -export type MemoryDreamOutcome = "completed" | "failed"; +export { normalizeMemoryHostEventRecordForStorage }; -/** Event emitted after a dreaming phase writes inline memory and/or reports. */ -export type MemoryHostDreamCompletedEvent = { - type: "memory.dream.completed"; - timestamp: string; - phase: MemoryDreamingPhaseName; - /** Missing on older event logs; readers should treat absent as "completed". */ - outcome?: MemoryDreamOutcome; - /** Error detail when outcome is "failed". */ - error?: string; - inlinePath?: string; - reportPath?: string; - lineCount: number; - storageMode: "inline" | "separate" | "both"; -}; - -/** Append-only memory host event schema stored as JSONL. */ -export type MemoryHostEvent = - | MemoryHostRecallRecordedEvent - | MemoryHostPromotionAppliedEvent - | MemoryHostDreamCompletedEvent; - -/** Full event-log record schema, including opt-in diagnostic variants. */ -export type MemoryHostEventRecord = MemoryHostEvent | MemoryHostRecallSkippedEvent; +/** Legacy workspace JSONL path retained only for doctor migration discovery. */ +export const MEMORY_HOST_EVENT_LOG_RELATIVE_PATH = path.join("memory", ".dreams", "events.jsonl"); -/** Resolve the event log path inside a workspace without touching the filesystem. */ +/** Resolve the retired JSONL source path without reading it at runtime. */ export function resolveMemoryHostEventLogPath(workspaceDir: string): string { return path.join(workspaceDir, MEMORY_HOST_EVENT_LOG_RELATIVE_PATH); } -/** Append one memory host event, creating the dreams directory with symlink-safe writes. */ +/** Append one memory host event to shared SQLite plugin state. */ export async function appendMemoryHostEvent( workspaceDir: string, event: MemoryHostEventRecord, + options: { env?: NodeJS.ProcessEnv } = {}, ): Promise { - const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir); - await fs.mkdir(path.dirname(eventLogPath), { recursive: true }); - await appendRegularFile({ - filePath: eventLogPath, - content: `${JSON.stringify(event)}\n`, - rejectSymlinkParents: true, + registerMemoryHostEvent({ + workspaceDir, + event, + ...(options.env ? { env: options.env } : {}), }); } -function parseMemoryHostEventRecord(line: string): MemoryHostEventRecord | null { - try { - const record = JSON.parse(line) as MemoryHostEventRecord; - if ( - record.type === "memory.recall.recorded" || - record.type === "memory.recall.skipped" || - record.type === "memory.promotion.applied" || - record.type === "memory.dream.completed" - ) { - return record; - } - } catch { - // The log is best-effort diagnostics; one malformed line must not hide - // later valid events or break memory status rendering. - } - return null; -} - async function readMemoryHostEventRecordsRaw(params: { workspaceDir: string; limit?: number; + env?: NodeJS.ProcessEnv; }): Promise { - const eventLogPath = resolveMemoryHostEventLogPath(params.workspaceDir); - const raw = await fs.readFile(eventLogPath, "utf8").catch((err: unknown) => { - if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { - return ""; - } - throw err; - }); - if (!raw.trim()) { - return []; - } - const events = raw - .split("\n") - .map((line) => line.trim()) - .filter(Boolean) - .flatMap((line) => { - const record = parseMemoryHostEventRecord(line); - return record ? [record] : []; - }); - if (!Number.isFinite(params.limit)) { - return events; - } - const limit = Math.max(0, Math.floor(params.limit as number)); - return limit === 0 ? [] : events.slice(-limit); + const events = listStoredMemoryHostEvents(params).map((entry) => entry.value.event); + return applyMemoryHostEventLimit(events, params.limit); } function applyMemoryHostEventLimit(events: T[], limit: number | undefined): T[] { @@ -155,12 +57,16 @@ function applyMemoryHostEventLimit(events: T[], limit: number | undefined): T return normalizedLimit === 0 ? [] : events.slice(-normalizedLimit); } -/** Read recent memory host events, ignoring corrupt JSONL lines left by partial writes. */ +/** Read recent memory host events, excluding opt-in diagnostic variants. */ export async function readMemoryHostEvents(params: { workspaceDir: string; limit?: number; + env?: NodeJS.ProcessEnv; }): Promise { - const events = await readMemoryHostEventRecordsRaw({ workspaceDir: params.workspaceDir }); + const events = await readMemoryHostEventRecordsRaw({ + workspaceDir: params.workspaceDir, + ...(params.env ? { env: params.env } : {}), + }); const legacyEvents = events.filter( (event): event is MemoryHostEvent => event.type !== "memory.recall.skipped", ); @@ -171,6 +77,7 @@ export async function readMemoryHostEvents(params: { export async function readMemoryHostEventRecords(params: { workspaceDir: string; limit?: number; + env?: NodeJS.ProcessEnv; }): Promise { return await readMemoryHostEventRecordsRaw(params); } diff --git a/src/plugin-sdk/memory-core-host-events.ts b/src/plugin-sdk/memory-core-host-events.ts index 2effe90b472..8e1c48a6b72 100644 --- a/src/plugin-sdk/memory-core-host-events.ts +++ b/src/plugin-sdk/memory-core-host-events.ts @@ -3,4 +3,19 @@ * `openclaw/plugin-sdk/memory-host-events` instead. */ -export * from "../memory-host-sdk/events.js"; +export { + appendMemoryHostEvent, + MEMORY_HOST_EVENT_LOG_RELATIVE_PATH, + readMemoryHostEventRecords, + readMemoryHostEvents, + resolveMemoryHostEventLogPath, +} from "../memory-host-sdk/events.js"; +export type { + MemoryDreamOutcome, + MemoryHostDreamCompletedEvent, + MemoryHostEvent, + MemoryHostEventRecord, + MemoryHostPromotionAppliedEvent, + MemoryHostRecallRecordedEvent, + MemoryHostRecallSkippedEvent, +} from "../memory-host-sdk/events.js"; diff --git a/src/plugin-sdk/memory-host-core.test.ts b/src/plugin-sdk/memory-host-core.test.ts index 801b5239265..3ed306fbb92 100644 --- a/src/plugin-sdk/memory-host-core.test.ts +++ b/src/plugin-sdk/memory-host-core.test.ts @@ -1,10 +1,15 @@ /** * Tests memory host core public artifact discovery and workspace handling. */ +import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createPluginStateKeyedStore, + resetPluginStateStoreForTests, +} from "../plugin-state/plugin-state-store.js"; import { clearMemoryPluginState, registerMemoryCapability, @@ -16,11 +21,14 @@ import { listMemoryHostPublicArtifacts, listActiveMemoryPublicArtifacts, } from "./memory-host-core.js"; -import { appendMemoryHostEvent, resolveMemoryHostEventLogPath } from "./memory-host-events.js"; +import { appendMemoryHostEvent } from "./memory-host-events.js"; describe("memory-host-core helpers", () => { afterEach(() => { clearMemoryPluginState(); + resetPluginStateStoreForTests(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); }); it("exposes the active memory prompt guidance builder for context engines", () => { @@ -68,9 +76,429 @@ describe("memory-host-core helpers", () => { ]); }); + it("propagates workspace inspection failures", async () => { + vi.spyOn(fs, "stat").mockRejectedValueOnce( + Object.assign(new Error("permission denied"), { code: "EACCES" }), + ); + + await expect( + listMemoryHostPublicArtifacts({ + cfg: { + agents: { + list: [{ id: "main", default: true, workspace: "/protected/workspace" }], + }, + }, + }), + ).rejects.toMatchObject({ code: "EACCES" }); + }); + + it("lists readable workspaces without requiring workspace-root writes", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-readonly-workspace-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + try { + vi.stubEnv("OPENCLAW_STATE_DIR", path.join(fixtureRoot, "state")); + await fs.mkdir(workspaceDir); + await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Read-only memory\n", "utf8"); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "read-only", + resultCount: 0, + results: [], + }); + await fs.chmod(workspaceDir, 0o500); + + await expect( + listMemoryHostPublicArtifacts({ + cfg: { + agents: { + list: [{ id: "main", default: true, workspace: workspaceDir }], + }, + }, + }), + ).resolves.toMatchObject([ + { + kind: "memory-root", + workspaceDir, + relativePath: "MEMORY.md", + }, + ]); + } finally { + await fs.chmod(workspaceDir, 0o700).catch(() => undefined); + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it.runIf(process.platform !== "win32")( + "propagates failures reading an existing event export", + async () => { + const fixtureRoot = await fs.mkdtemp( + path.join(os.tmpdir(), "memory-host-unreadable-export-"), + ); + const workspaceDir = path.join(fixtureRoot, "workspace"); + let eventExportPath: string | undefined; + try { + vi.stubEnv("OPENCLAW_STATE_DIR", path.join(fixtureRoot, "state")); + await fs.mkdir(workspaceDir); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "unreadable export", + resultCount: 0, + results: [], + }); + const firstListing = await listMemoryHostPublicArtifacts({ + cfg: { + agents: { + list: [{ id: "main", default: true, workspace: workspaceDir }], + }, + }, + }); + eventExportPath = firstListing.find( + (artifact) => artifact.kind === "event-log", + )?.absolutePath; + if (!eventExportPath) { + throw new Error("expected memory event export"); + } + await fs.chmod(eventExportPath, 0o000); + + await expect( + listMemoryHostPublicArtifacts({ + cfg: { + agents: { + list: [{ id: "main", default: true, workspace: workspaceDir }], + }, + }, + }), + ).rejects.toThrow(); + } finally { + if (eventExportPath) { + await fs.chmod(eventExportPath, 0o600).catch(() => undefined); + } + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "does not delete an event export through a symlinked parent", + async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-symlink-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const externalMemoryDir = path.join(fixtureRoot, "external-memory"); + const stateDir = path.join(fixtureRoot, "state"); + try { + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + await fs.mkdir(workspaceDir); + await fs.mkdir(stateDir); + const stateHash = createHash("sha256") + .update(await fs.realpath(stateDir)) + .digest("hex") + .slice(0, 32); + const externalExport = path.join( + externalMemoryDir, + "events", + stateHash, + "memory-host-events.jsonl", + ); + const externalOwner = path.join( + path.dirname(externalExport), + ".openclaw-memory-host-events-owner.json", + ); + await fs.mkdir(path.dirname(externalExport), { recursive: true }); + await fs.writeFile(externalExport, '{"type":"external"}\n', "utf8"); + await fs.writeFile(externalOwner, '{"kind":"external"}\n', "utf8"); + await fs.symlink(externalMemoryDir, path.join(workspaceDir, "memory")); + + await expect( + listMemoryHostPublicArtifacts({ + cfg: { + agents: { + list: [{ id: "main", default: true, workspace: workspaceDir }], + }, + }, + }), + ).resolves.toEqual([]); + await expect(fs.readFile(externalExport, "utf8")).resolves.toBe('{"type":"external"}\n'); + await expect(fs.readFile(externalOwner, "utf8")).resolves.toBe('{"kind":"external"}\n'); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }, + ); + + it("does not replace or delete an unowned event export path", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-unowned-export-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + try { + vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot); + await fs.mkdir(workspaceDir); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "must not replace user file", + resultCount: 0, + results: [], + }); + const stateHash = createHash("sha256") + .update(await fs.realpath(fixtureRoot)) + .digest("hex") + .slice(0, 32); + const exportPath = path.join( + workspaceDir, + "memory", + "events", + stateHash, + "memory-host-events.jsonl", + ); + await fs.mkdir(path.dirname(exportPath), { recursive: true }); + await fs.writeFile(exportPath, '{"owner":"user"}\n', "utf8"); + + const listed = await listMemoryHostPublicArtifacts({ + cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } }, + }); + expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(false); + await expect(fs.readFile(exportPath, "utf8")).resolves.toBe('{"owner":"user"}\n'); + + await createPluginStateKeyedStore("memory-core", { + namespace: "memory-host.events", + maxEntries: 10_000, + env: { ...process.env, OPENCLAW_STATE_DIR: fixtureRoot }, + }).clear(); + await listMemoryHostPublicArtifacts({ + cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } }, + }); + await expect(fs.readFile(exportPath, "utf8")).resolves.toBe('{"owner":"user"}\n'); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("does not let an orphaned owner marker authorize a later workspace file", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-orphan-owner-")); + const stateDir = path.join(fixtureRoot, "state"); + const workspaceDir = path.join(fixtureRoot, "workspace"); + try { + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + await fs.mkdir(workspaceDir); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "expected export", + resultCount: 0, + results: [], + }); + const stateHash = createHash("sha256") + .update(await fs.realpath(stateDir)) + .digest("hex") + .slice(0, 32); + const workspaceHash = createHash("sha256") + .update(await fs.realpath(workspaceDir)) + .digest("hex") + .slice(0, 32); + const exportDir = path.join(workspaceDir, "memory", "events", stateHash); + const exportPath = path.join(exportDir, "memory-host-events.jsonl"); + const ownerPath = path.join(exportDir, ".openclaw-memory-host-events-owner.json"); + const expectedContent = `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "expected export", + resultCount: 0, + results: [], + })}\n`; + await fs.mkdir(exportDir, { recursive: true }); + await fs.writeFile( + ownerPath, + `${JSON.stringify({ + schemaVersion: 3, + kind: "openclaw-memory-host-events-export", + stateHash, + workspaceHash, + pendingContentSha256: createHash("sha256").update(expectedContent).digest("hex"), + })}\n`, + "utf8", + ); + await fs.writeFile(exportPath, '{"owner":"user after crash"}\n', "utf8"); + + const listed = await listMemoryHostPublicArtifacts({ + cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } }, + }); + + expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(false); + await expect(fs.readFile(exportPath, "utf8")).resolves.toBe('{"owner":"user after crash"}\n'); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("does not claim a same-content inode that replaces an exclusive export", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-create-replace-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const cfg = { + agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] }, + }; + const event = { + type: "memory.recall.recorded" as const, + timestamp: "2026-05-18T12:00:00.000Z", + query: "exclusive create", + resultCount: 0, + results: [], + }; + const expectedContent = `${JSON.stringify(event)}\n`; + const originalOpen = fs.open.bind(fs); + let exportOpenCount = 0; + try { + vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot); + await fs.mkdir(workspaceDir, { recursive: true }); + await appendMemoryHostEvent(workspaceDir, event); + const stateHash = createHash("sha256") + .update(await fs.realpath(fixtureRoot)) + .digest("hex") + .slice(0, 32); + const exportPath = path.join( + workspaceDir, + "memory", + "events", + stateHash, + "memory-host-events.jsonl", + ); + vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const target = args[0]; + if (typeof target === "string" && path.resolve(target) === path.resolve(exportPath)) { + exportOpenCount += 1; + if (exportOpenCount === 4) { + await fs.rename(exportPath, `${exportPath}.openclaw-created`); + const replacement = await originalOpen(exportPath, "wx", 0o600); + try { + await replacement.writeFile(expectedContent, "utf8"); + } finally { + await replacement.close(); + } + } + } + return await originalOpen(...args); + }); + + const racedArtifacts = await listMemoryHostPublicArtifacts({ cfg }); + expect(racedArtifacts.some((artifact) => artifact.kind === "event-log")).toBe(false); + await expect(fs.readFile(exportPath, "utf8")).resolves.toBe(expectedContent); + + vi.restoreAllMocks(); + await appendMemoryHostEvent(workspaceDir, { ...event, query: "must stay foreign" }); + const retriedArtifacts = await listMemoryHostPublicArtifacts({ cfg }); + expect(retriedArtifacts.some((artifact) => artifact.kind === "event-log")).toBe(false); + await expect(fs.readFile(exportPath, "utf8")).resolves.toBe(expectedContent); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("does not claim a hash-only pending event export after interruption", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-pending-owner-")); + const stateDir = path.join(fixtureRoot, "state"); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const firstEvent = { + type: "memory.recall.recorded" as const, + timestamp: "2026-05-18T12:00:00.000Z", + query: "pending export", + resultCount: 0, + results: [], + }; + try { + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + await fs.mkdir(workspaceDir); + await appendMemoryHostEvent(workspaceDir, firstEvent); + const stateHash = createHash("sha256") + .update(await fs.realpath(stateDir)) + .digest("hex") + .slice(0, 32); + const workspaceHash = createHash("sha256") + .update(await fs.realpath(workspaceDir)) + .digest("hex") + .slice(0, 32); + const exportDir = path.join(workspaceDir, "memory", "events", stateHash); + const exportPath = path.join(exportDir, "memory-host-events.jsonl"); + const ownerPath = path.join(exportDir, ".openclaw-memory-host-events-owner.json"); + const pendingContent = `${JSON.stringify(firstEvent)}\n`; + await fs.mkdir(exportDir, { recursive: true }); + await fs.writeFile(exportPath, pendingContent, "utf8"); + await fs.writeFile( + ownerPath, + `${JSON.stringify({ + schemaVersion: 3, + kind: "openclaw-memory-host-events-export", + stateHash, + workspaceHash, + pendingContentSha256: createHash("sha256").update(pendingContent).digest("hex"), + })}\n`, + "utf8", + ); + await appendMemoryHostEvent(workspaceDir, { + ...firstEvent, + timestamp: "2026-05-18T12:01:00.000Z", + query: "after interruption", + }); + + const listed = await listMemoryHostPublicArtifacts({ + cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } }, + }); + + expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(false); + await expect(fs.readFile(exportPath, "utf8")).resolves.toBe(pendingContent); + const owner = JSON.parse(await fs.readFile(ownerPath, "utf8")) as { + contentSha256?: string; + fileDev?: string; + fileIno?: string; + pendingContentSha256?: string; + }; + expect(owner.pendingContentSha256).toBe( + createHash("sha256") + .update(await fs.readFile(exportPath)) + .digest("hex"), + ); + expect(owner.contentSha256).toBeUndefined(); + expect(owner.fileDev).toBeUndefined(); + expect(owner.fileIno).toBeUndefined(); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("omits the event export when a workspace path component is a file", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-blocked-export-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + try { + vi.stubEnv("OPENCLAW_STATE_DIR", path.join(fixtureRoot, "state")); + await fs.mkdir(workspaceDir); + await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Still visible\n", "utf8"); + await fs.writeFile(path.join(workspaceDir, "memory"), "user file\n", "utf8"); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "blocked export", + resultCount: 0, + results: [], + }); + + await expect( + listMemoryHostPublicArtifacts({ + cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } }, + }), + ).resolves.toEqual([ + expect.objectContaining({ kind: "memory-root", relativePath: "MEMORY.md" }), + ]); + await expect(fs.readFile(path.join(workspaceDir, "memory"), "utf8")).resolves.toBe( + "user file\n", + ); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + it("lists shared public artifacts from memory workspaces", async () => { const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-public-artifacts-")); try { + vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot); const workspaceDir = path.join(fixtureRoot, "workspace"); await fs.mkdir(path.join(workspaceDir, "memory", "dreaming"), { recursive: true }); await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Durable Memory\n", "utf8"); @@ -84,6 +512,8 @@ describe("memory-host-core helpers", () => { "# Dream Report\n", "utf8", ); + const eventStoredAt = Date.parse("2026-05-19T09:30:00.000Z"); + vi.spyOn(Date, "now").mockReturnValue(eventStoredAt); await appendMemoryHostEvent(workspaceDir, { type: "memory.recall.recorded", timestamp: "2026-05-18T12:00:00.000Z", @@ -92,15 +522,18 @@ describe("memory-host-core helpers", () => { results: [], }); - await expect( - listMemoryHostPublicArtifacts({ - cfg: { - agents: { - list: [{ id: "main", default: true, workspace: workspaceDir }], - }, + const artifacts = await listMemoryHostPublicArtifacts({ + cfg: { + agents: { + list: [{ id: "main", default: true, workspace: workspaceDir }], }, - }), - ).resolves.toEqual([ + }, + }); + const eventArtifact = artifacts.find((artifact) => artifact.kind === "event-log"); + if (!eventArtifact) { + throw new Error("expected memory event export"); + } + expect(artifacts.filter((artifact) => artifact.kind !== "event-log")).toEqual([ { kind: "memory-root", workspaceDir, @@ -125,15 +558,437 @@ describe("memory-host-core helpers", () => { agentIds: ["main"], contentType: "markdown", }, - { - kind: "event-log", - workspaceDir, - relativePath: "memory/.dreams/events.jsonl", - absolutePath: resolveMemoryHostEventLogPath(workspaceDir), - agentIds: ["main"], - contentType: "json", - }, ]); + expect(eventArtifact).toMatchObject({ + kind: "event-log", + workspaceDir, + agentIds: ["main"], + contentType: "json", + }); + expect(eventArtifact.relativePath).toMatch( + /^memory\/events\/[a-f0-9]{32}\/memory-host-events\.jsonl$/u, + ); + expect(eventArtifact.absolutePath).toBe( + path.join(workspaceDir, ...eventArtifact.relativePath.split("/")), + ); + const eventExportPath = eventArtifact.absolutePath; + await expect(fs.readFile(eventExportPath, "utf8")).resolves.toBe( + `${JSON.stringify({ + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "bridge", + resultCount: 0, + results: [], + })}\n`, + ); + const exportStat = await fs.stat(eventExportPath); + const exportOwner = JSON.parse( + await fs.readFile( + path.join(path.dirname(eventExportPath), ".openclaw-memory-host-events-owner.json"), + "utf8", + ), + ) as { fileDev?: string; fileIno?: string }; + expect(exportOwner.fileDev).toBe(String(exportStat.dev)); + expect(exportOwner.fileIno).toBe(String(exportStat.ino)); + await expect( + fs.access(path.join(workspaceDir, ".memory-host-events-export.lock")), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect( + (await fs.readdir(fixtureRoot)).some((entry) => + entry.startsWith(".memory-host-events-export-"), + ), + ).toBe(false); + + await createPluginStateKeyedStore("memory-core", { + namespace: "memory-host.events", + maxEntries: 10_000, + env: { ...process.env, OPENCLAW_STATE_DIR: fixtureRoot }, + }).clear(); + const afterRetention = await listMemoryHostPublicArtifacts({ + cfg: { + agents: { + list: [{ id: "main", default: true, workspace: workspaceDir }], + }, + }, + }); + expect(afterRetention.some((artifact) => artifact.kind === "event-log")).toBe(false); + await expect(fs.readFile(eventExportPath, "utf8")).resolves.toBe(""); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it.runIf(process.platform !== "win32")( + "does not let a workspace alias overwrite a newer event export", + async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-race-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const workspaceAlias = path.join(fixtureRoot, "workspace-alias"); + const cfg = { + agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] }, + }; + const aliasCfg = { + agents: { list: [{ id: "main", default: true, workspace: workspaceAlias }] }, + }; + const originalOpen = fs.open.bind(fs); + let releaseFirstRead: (() => void) | undefined; + let signalFirstRead: (() => void) | undefined; + let shouldBlockFirstRead = true; + const firstReadStarted = new Promise((resolve) => { + signalFirstRead = resolve; + }); + const openSpy = vi + .spyOn(fs, "open") + .mockImplementation(async (...args: Parameters) => { + const target = args[0]; + if ( + shouldBlockFirstRead && + typeof target === "string" && + path.basename(target) === "memory-host-events.jsonl" && + path.resolve(target).startsWith(`${path.resolve(workspaceDir)}${path.sep}`) + ) { + shouldBlockFirstRead = false; + signalFirstRead?.(); + await new Promise((resolve) => { + releaseFirstRead = resolve; + }); + } + return await originalOpen(...args); + }); + + try { + vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot); + await fs.mkdir(workspaceDir, { recursive: true }); + await fs.symlink(workspaceDir, workspaceAlias); + await appendMemoryHostEvent(workspaceAlias, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "older", + resultCount: 0, + results: [], + }); + const olderListing = listMemoryHostPublicArtifacts({ cfg }); + await firstReadStarted; + + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:01:00.000Z", + query: "newer", + resultCount: 0, + results: [], + }); + const newerListing = listMemoryHostPublicArtifacts({ cfg: aliasCfg }); + await Promise.race([ + newerListing, + new Promise((resolve) => { + setTimeout(resolve, 250); + }), + ]); + releaseFirstRead?.(); + const [olderArtifacts, newerArtifacts] = await Promise.all([olderListing, newerListing]); + const olderExport = olderArtifacts.find((artifact) => artifact.kind === "event-log"); + const newerExport = newerArtifacts.find((artifact) => artifact.kind === "event-log"); + expect(olderExport?.absolutePath).toBe(newerExport?.absolutePath); + const eventExportPath = newerExport?.absolutePath; + if (!eventExportPath) { + throw new Error("expected memory event export"); + } + + const exported = (await fs.readFile(eventExportPath, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { query: string }); + expect(exported.map((event) => event.query)).toEqual(["older", "newer"]); + } finally { + releaseFirstRead?.(); + openSpy.mockRestore(); + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }, + ); + + it.each([ + { name: "refresh", clearEvents: false }, + { name: "retirement", clearEvents: true }, + ])("preserves a replacement installed during event export $name", async ({ clearEvents }) => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-replace-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const cfg = { + agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] }, + }; + const replacement = '{"owner":"workspace"}\n'; + const originalOpen = fs.open.bind(fs); + let exportOpenCount = 0; + let eventExportPath: string | undefined; + try { + vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot); + await fs.mkdir(workspaceDir, { recursive: true }); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "first", + resultCount: 0, + results: [], + }); + eventExportPath = (await listMemoryHostPublicArtifacts({ cfg })).find( + (artifact) => artifact.kind === "event-log", + )?.absolutePath; + if (!eventExportPath) { + throw new Error("expected memory event export"); + } + if (clearEvents) { + await createPluginStateKeyedStore("memory-core", { + namespace: "memory-host.events", + maxEntries: 10_000, + env: { ...process.env, OPENCLAW_STATE_DIR: fixtureRoot }, + }).clear(); + } else { + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:01:00.000Z", + query: "second", + resultCount: 0, + results: [], + }); + } + + const expectedExportPath = path.resolve(eventExportPath); + vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const target = args[0]; + if (typeof target === "string" && path.resolve(target) === expectedExportPath) { + exportOpenCount += 1; + if (exportOpenCount === 2) { + await fs.rename(expectedExportPath, `${expectedExportPath}.openclaw-owned`); + await fs.writeFile(expectedExportPath, replacement, "utf8"); + } + } + return await originalOpen(...args); + }); + + const artifacts = await listMemoryHostPublicArtifacts({ cfg }); + expect(artifacts.some((artifact) => artifact.kind === "event-log")).toBe(false); + await expect(fs.readFile(expectedExportPath, "utf8")).resolves.toBe(replacement); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it.each([ + { name: "refresh", clearEvents: false }, + { name: "retirement", clearEvents: true }, + ])("keeps ownership through same-inode event export $name", async ({ clearEvents }) => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-inode-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const cfg = { + agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] }, + }; + const originalOpen = fs.open.bind(fs); + let exportOpenCount = 0; + try { + vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot); + await fs.mkdir(workspaceDir, { recursive: true }); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "first", + resultCount: 0, + results: [], + }); + const eventExportPath = (await listMemoryHostPublicArtifacts({ cfg })).find( + (artifact) => artifact.kind === "event-log", + )?.absolutePath; + if (!eventExportPath) { + throw new Error("expected memory event export"); + } + if (clearEvents) { + await createPluginStateKeyedStore("memory-core", { + namespace: "memory-host.events", + maxEntries: 10_000, + env: { ...process.env, OPENCLAW_STATE_DIR: fixtureRoot }, + }).clear(); + } else { + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:01:00.000Z", + query: "second", + resultCount: 0, + results: [], + }); + } + + const expectedExportPath = path.resolve(eventExportPath); + const originalStat = await fs.stat(expectedExportPath); + vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { + const target = args[0]; + if (typeof target === "string" && path.resolve(target) === expectedExportPath) { + exportOpenCount += 1; + if (exportOpenCount === 2) { + const writer = await originalOpen(expectedExportPath, "w"); + try { + await writer.writeFile('{"owner":"same inode"}\n', "utf8"); + } finally { + await writer.close(); + } + } + } + return await originalOpen(...args); + }); + + const artifacts = await listMemoryHostPublicArtifacts({ cfg }); + expect((await fs.stat(expectedExportPath)).ino).toBe(originalStat.ino); + if (clearEvents) { + expect(artifacts.some((artifact) => artifact.kind === "event-log")).toBe(false); + await expect(fs.readFile(expectedExportPath, "utf8")).resolves.toBe(""); + } else { + expect(artifacts.some((artifact) => artifact.kind === "event-log")).toBe(true); + await expect(fs.readFile(expectedExportPath, "utf8")).resolves.toContain("second"); + } + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("retries an owned event export after a same-inode post-write race", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-retry-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const cfg = { + agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] }, + }; + const originalOpen = fs.open.bind(fs); + let exportOpenCount = 0; + try { + vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot); + await fs.mkdir(workspaceDir, { recursive: true }); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "first", + resultCount: 0, + results: [], + }); + const eventExportPath = (await listMemoryHostPublicArtifacts({ cfg })).find( + (artifact) => artifact.kind === "event-log", + )?.absolutePath; + if (!eventExportPath) { + throw new Error("expected memory event export"); + } + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:01:00.000Z", + query: "second", + resultCount: 0, + results: [], + }); + + const expectedExportPath = path.resolve(eventExportPath); + const openSpy = vi + .spyOn(fs, "open") + .mockImplementation(async (...args: Parameters) => { + const target = args[0]; + if (typeof target === "string" && path.resolve(target) === expectedExportPath) { + exportOpenCount += 1; + if (exportOpenCount === 4) { + const writer = await originalOpen(expectedExportPath, "w"); + try { + await writer.writeFile('{"owner":"post-write race"}\n', "utf8"); + } finally { + await writer.close(); + } + } + } + return await originalOpen(...args); + }); + + const racedArtifacts = await listMemoryHostPublicArtifacts({ cfg }); + expect(racedArtifacts.some((artifact) => artifact.kind === "event-log")).toBe(false); + openSpy.mockRestore(); + + const retriedArtifacts = await listMemoryHostPublicArtifacts({ cfg }); + expect(retriedArtifacts.some((artifact) => artifact.kind === "event-log")).toBe(true); + await expect(fs.readFile(expectedExportPath, "utf8")).resolves.toContain("second"); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("repairs an oversized owned event export by inode", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-oversized-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const cfg = { + agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] }, + }; + try { + vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot); + await fs.mkdir(workspaceDir, { recursive: true }); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: "first", + resultCount: 0, + results: [], + }); + const eventExportPath = (await listMemoryHostPublicArtifacts({ cfg })).find( + (artifact) => artifact.kind === "event-log", + )?.absolutePath; + if (!eventExportPath) { + throw new Error("expected memory event export"); + } + const originalStat = await fs.stat(eventExportPath); + await fs.appendFile(eventExportPath, "x".repeat(1024 * 1024 + 1), "utf8"); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:01:00.000Z", + query: "second", + resultCount: 0, + results: [], + }); + + const artifacts = await listMemoryHostPublicArtifacts({ cfg }); + expect(artifacts.some((artifact) => artifact.kind === "event-log")).toBe(true); + expect((await fs.stat(eventExportPath)).ino).toBe(originalStat.ino); + expect((await fs.stat(eventExportPath)).size).toBeLessThan(1024 * 1024); + await expect(fs.readFile(eventExportPath, "utf8")).resolves.toContain("second"); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("keeps public event exports isolated across state directories", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-owner-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const cfg = { + agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] }, + }; + try { + await fs.mkdir(workspaceDir, { recursive: true }); + + const exports: string[] = []; + for (const profile of ["profile-a", "profile-b"]) { + vi.stubEnv("OPENCLAW_STATE_DIR", path.join(fixtureRoot, profile)); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: "2026-05-18T12:00:00.000Z", + query: profile, + resultCount: 0, + results: [], + }); + const artifact = (await listMemoryHostPublicArtifacts({ cfg })).find( + (candidate) => candidate.kind === "event-log", + ); + if (!artifact) { + throw new Error("expected memory event export"); + } + exports.push(artifact.absolutePath); + await expect(fs.readFile(artifact.absolutePath, "utf8")).resolves.toContain(profile); + } + + expect(new Set(exports).size).toBe(2); + const [profileAExport, profileBExport] = exports; + if (!profileAExport || !profileBExport) { + throw new Error("expected state-qualified memory event exports"); + } + await expect(fs.readFile(profileAExport, "utf8")).resolves.toContain("profile-a"); + await expect(fs.readFile(profileBExport, "utf8")).resolves.toContain("profile-b"); } finally { await fs.rm(fixtureRoot, { recursive: true, force: true }); } diff --git a/src/plugin-sdk/memory-host-core.ts b/src/plugin-sdk/memory-host-core.ts index 632989c0c64..1fa94858ce1 100644 --- a/src/plugin-sdk/memory-host-core.ts +++ b/src/plugin-sdk/memory-host-core.ts @@ -4,9 +4,208 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { OpenClawConfig } from "../config/config.js"; +import { resolveStateDir } from "../config/paths.js"; +import { sha256Hex, sha256HexPrefix } from "../infra/crypto-digest.js"; +import { withFileLock } from "../infra/file-lock.js"; +import { sameFileIdentity, type FileIdentityStat } from "../infra/fs-safe-advanced.js"; +import { FsSafeError, root as createFsSafeRoot } from "../infra/fs-safe.js"; +import { syncDirectoryBestEffort } from "../infra/sqlite-snapshot.js"; +import { + MAX_MEMORY_HOST_PUBLIC_EXPORT_BYTES, + serializeMemoryHostEventExport, +} from "../memory-host-sdk/event-export.js"; +import { listStoredMemoryHostEvents } from "../memory-host-sdk/event-store.js"; import type { MemoryPluginPublicArtifact } from "../plugins/memory-state.js"; +import { KeyedAsyncQueue } from "./keyed-async-queue.js"; import { resolveMemoryDreamingWorkspaces } from "./memory-core-host-status.js"; -import { resolveMemoryHostEventLogPath } from "./memory-host-events.js"; +import { + isMemoryHostEventArtifactAtIdentity, + isMissingPathError, + isRejectedWorkspaceArtifactPath, + memoryHostEventExportOwnerContent, + publishMemoryHostEventArtifact, + rewriteMemoryHostEventArtifactIfUnchanged, +} from "./memory-host-event-export.js"; + +const MEMORY_HOST_EVENTS_FILENAME = "memory-host-events.jsonl"; +const MEMORY_HOST_EVENTS_OWNER_FILENAME = ".openclaw-memory-host-events-owner.json"; +const MAX_MEMORY_HOST_PUBLIC_EXPORT_EVENTS = 1_000; +const MEMORY_HOST_EVENT_EXPORT_LOCK_OPTIONS = { + retries: { retries: 20, factor: 1.3, minTimeout: 25, maxTimeout: 250, randomize: true }, + stale: 30_000, +} as const; +const memoryHostEventExportQueue = new KeyedAsyncQueue(); + +function isWorkspaceWriteUnavailable(error: unknown, seen = new Set()): boolean { + if (!error || typeof error !== "object" || seen.has(error)) { + return false; + } + seen.add(error); + const code = (error as { code?: unknown }).code; + if ( + code === "EACCES" || + code === "EEXIST" || + code === "ENOTDIR" || + code === "EPERM" || + code === "EROFS" || + (error instanceof FsSafeError && (code === "not-file" || code === "not-removable")) + ) { + return true; + } + if (error instanceof FsSafeError && error.category === "policy" && code !== "invalid-path") { + return false; + } + return isWorkspaceWriteUnavailable((error as { cause?: unknown }).cause, seen); +} + +async function resolveMemoryHostEventExportOwner(workspaceDir: string): Promise<{ + queueKey: string; + lockTarget: string; + relativePath: string; + ownerRelativePath: string; + stateHash: string; + workspaceHash: string; +}> { + const requestedStateDir = path.resolve(resolveStateDir()); + await fs.mkdir(requestedStateDir, { recursive: true, mode: 0o700 }); + const stateDir = await fs.realpath(requestedStateDir); + const stateHash = sha256HexPrefix(stateDir, 32); + const workspaceHash = sha256HexPrefix(path.resolve(workspaceDir), 32); + const exportDirectory = path.posix.join("memory", "events", stateHash); + return { + queueKey: `${stateHash}\0${workspaceHash}`, + lockTarget: path.join(stateDir, `.memory-host-events-export-${workspaceHash}`), + relativePath: path.posix.join(exportDirectory, MEMORY_HOST_EVENTS_FILENAME), + ownerRelativePath: path.posix.join(exportDirectory, MEMORY_HOST_EVENTS_OWNER_FILENAME), + stateHash, + workspaceHash, + }; +} + +async function readMemoryHostEventExportOwnership( + workspaceRoot: Awaited>, + owner: Awaited>, +): Promise< + | { + kind: "owned"; + content: string | undefined; + identity: FileIdentityStat; + ownerContent: string; + needsFinalize: boolean; + } + | { kind: "missing" } + | { kind: "orphan"; ownerContent: string } + | { kind: "pending-missing"; ownerContent: string } + | { kind: "foreign" } +> { + const content = await workspaceRoot.readText(owner.ownerRelativePath).catch((error: unknown) => { + if (isMissingPathError(error)) { + return undefined; + } + if (isRejectedWorkspaceArtifactPath(error)) { + return null; + } + throw error; + }); + if (content === null) { + return { kind: "foreign" }; + } + if (content === undefined) { + return { kind: "missing" }; + } + let parsed: unknown; + try { + parsed = JSON.parse(content) as unknown; + } catch { + return { kind: "foreign" }; + } + if ( + !parsed || + typeof parsed !== "object" || + Array.isArray(parsed) || + (parsed as { schemaVersion?: unknown }).schemaVersion !== 3 || + (parsed as { kind?: unknown }).kind !== "openclaw-memory-host-events-export" || + (parsed as { stateHash?: unknown }).stateHash !== owner.stateHash || + (parsed as { workspaceHash?: unknown }).workspaceHash !== owner.workspaceHash || + ((parsed as { contentSha256?: unknown }).contentSha256 !== undefined && + typeof (parsed as { contentSha256?: unknown }).contentSha256 !== "string") || + ((parsed as { pendingContentSha256?: unknown }).pendingContentSha256 !== undefined && + typeof (parsed as { pendingContentSha256?: unknown }).pendingContentSha256 !== "string") || + ((parsed as { contentSha256?: unknown }).contentSha256 === undefined && + (parsed as { pendingContentSha256?: unknown }).pendingContentSha256 === undefined) || + ((parsed as { fileDev?: unknown }).fileDev === undefined) !== + ((parsed as { fileIno?: unknown }).fileIno === undefined) || + ((parsed as { fileDev?: unknown }).fileDev !== undefined && + (typeof (parsed as { fileDev?: unknown }).fileDev !== "string" || + !/^\d+$/u.test((parsed as { fileDev: string }).fileDev) || + typeof (parsed as { fileIno?: unknown }).fileIno !== "string" || + !/^\d+$/u.test((parsed as { fileIno: string }).fileIno))) + ) { + return { kind: "foreign" }; + } + const storedIdentity = + typeof (parsed as { fileDev?: unknown }).fileDev === "string" && + typeof (parsed as { fileIno?: unknown }).fileIno === "string" + ? { + dev: BigInt((parsed as { fileDev: string }).fileDev), + ino: BigInt((parsed as { fileIno: string }).fileIno), + } + : undefined; + let openedExport: Awaited> | undefined; + try { + openedExport = await workspaceRoot.open(owner.relativePath); + } catch (error) { + if (isMissingPathError(error)) { + openedExport = undefined; + } else if (isRejectedWorkspaceArtifactPath(error)) { + return { kind: "foreign" }; + } else { + throw error; + } + } + if (!openedExport) { + return typeof (parsed as { pendingContentSha256?: unknown }).pendingContentSha256 === "string" + ? { kind: "pending-missing", ownerContent: content } + : { kind: "orphan", ownerContent: content }; + } + let exportContent: string; + const exportIdentity: FileIdentityStat = { + dev: openedExport.stat.dev, + ino: openedExport.stat.ino, + }; + const identityOwned = + storedIdentity !== undefined && sameFileIdentity(storedIdentity, exportIdentity); + try { + if (openedExport.stat.size > MAX_MEMORY_HOST_PUBLIC_EXPORT_BYTES) { + return identityOwned + ? { + kind: "owned", + content: undefined, + identity: exportIdentity, + ownerContent: content, + needsFinalize: true, + } + : { kind: "foreign" }; + } + exportContent = await openedExport.handle.readFile({ encoding: "utf8" }); + } finally { + await openedExport.handle.close().catch(() => undefined); + } + const exportSha256 = sha256Hex(exportContent); + const currentSha256 = (parsed as { contentSha256?: string }).contentSha256; + const pendingSha256 = (parsed as { pendingContentSha256?: string }).pendingContentSha256; + // Hash-only markers never reach the owned branch; only the persisted inode + // identity authorizes later mutation of this workspace artifact. + return identityOwned + ? { + kind: "owned", + content: exportContent, + identity: exportIdentity, + ownerContent: content, + needsFinalize: exportSha256 !== currentSha256 || pendingSha256 !== undefined, + } + : { kind: "foreign" }; +} export { buildMemoryPromptSection as buildActiveMemoryPromptSection, @@ -25,15 +224,6 @@ export { resolveDefaultAgentId } from "../agents/agent-scope-config.js"; export { resolveSessionAgentId } from "../agents/agent-scope.js"; export { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; -async function pathExists(filePath: string): Promise { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } -} - async function listMarkdownFilesRecursive(rootDir: string): Promise { const entries = await fs.readdir(rootDir, { withFileTypes: true }).catch(() => []); const files: string[] = []; @@ -50,6 +240,215 @@ async function listMarkdownFilesRecursive(rootDir: string): Promise { return files.toSorted((left, right) => left.localeCompare(right)); } +async function materializeMemoryHostEventExport(params: { + workspaceDir: string; +}): Promise<{ absolutePath: string; relativePath: string } | undefined> { + const requestedWorkspace = path.resolve(params.workspaceDir); + const workspace = await fs.stat(requestedWorkspace).catch((error: unknown) => { + if (isMissingPathError(error)) { + return undefined; + } + throw error; + }); + if (!workspace?.isDirectory()) { + return undefined; + } + const workspaceRoot = await createFsSafeRoot(requestedWorkspace, { + hardlinks: "reject", + mkdir: true, + mode: 0o600, + symlinks: "reject", + }); + const workspaceKey = workspaceRoot.rootReal; + const owner = await resolveMemoryHostEventExportOwner(workspaceKey); + // The queue handles re-entrant calls in this process; the sidecar lock makes + // snapshot, cleanup, and replacement one ordered operation across processes. + // State-qualified paths keep different profiles from replacing each other's export. + return memoryHostEventExportQueue.enqueue(owner.queueKey, async () => { + const absolutePath = path.join(workspaceKey, ...owner.relativePath.split("/")); + return await withFileLock(owner.lockTarget, MEMORY_HOST_EVENT_EXPORT_LOCK_OPTIONS, async () => { + const storedEvents = listStoredMemoryHostEvents({ + workspaceDir: workspaceKey, + limit: MAX_MEMORY_HOST_PUBLIC_EXPORT_EVENTS, + }); + const ownership = await readMemoryHostEventExportOwnership(workspaceRoot, owner); + if (ownership.kind === "foreign") { + return undefined; + } + if (storedEvents.length === 0 && ownership.kind !== "owned") { + return undefined; + } + const content = storedEvents.length > 0 ? serializeMemoryHostEventExport(storedEvents) : ""; + const contentSha256 = sha256Hex(content); + let publishedIdentity: FileIdentityStat | undefined; + if (ownership.kind === "missing") { + const existing = await workspaceRoot + .readText(owner.relativePath) + .catch((error: unknown) => { + if (isMissingPathError(error)) { + return undefined; + } + if (isRejectedWorkspaceArtifactPath(error)) { + return null; + } + throw error; + }); + if (existing !== undefined) { + return undefined; + } + try { + const pendingOwnerContent = memoryHostEventExportOwnerContent(owner, { + pendingSha256: contentSha256, + }); + await workspaceRoot.create(owner.ownerRelativePath, pendingOwnerContent, { + mkdir: true, + mode: 0o600, + }); + await syncDirectoryBestEffort(path.dirname(absolutePath)); + publishedIdentity = await publishMemoryHostEventArtifact({ + workspaceRoot, + owner, + absolutePath, + expectedOwnerContent: pendingOwnerContent, + content, + contentSha256, + }); + if (!publishedIdentity) { + return undefined; + } + } catch (error) { + if (isWorkspaceWriteUnavailable(error)) { + return undefined; + } + throw error; + } + } else if (ownership.kind === "pending-missing" || ownership.kind === "orphan") { + try { + const pendingOwnerContent = memoryHostEventExportOwnerContent(owner, { + pendingSha256: contentSha256, + }); + if ( + !(await rewriteMemoryHostEventArtifactIfUnchanged({ + workspaceRoot, + relativePath: owner.ownerRelativePath, + expectedContent: ownership.ownerContent, + nextContent: pendingOwnerContent, + })) + ) { + return undefined; + } + await syncDirectoryBestEffort(path.dirname(absolutePath)); + publishedIdentity = await publishMemoryHostEventArtifact({ + workspaceRoot, + owner, + absolutePath, + expectedOwnerContent: pendingOwnerContent, + content, + contentSha256, + }); + if (!publishedIdentity) { + return undefined; + } + } catch (error) { + if (isWorkspaceWriteUnavailable(error)) { + return undefined; + } + throw error; + } + } else if (ownership.content !== content) { + publishedIdentity = ownership.identity; + try { + const updateOwnerContent = memoryHostEventExportOwnerContent(owner, { + pendingSha256: contentSha256, + identity: ownership.identity, + ...(ownership.content === undefined + ? {} + : { currentSha256: sha256Hex(ownership.content) }), + }); + const currentOwnerContent = memoryHostEventExportOwnerContent(owner, { + currentSha256: contentSha256, + identity: ownership.identity, + }); + if ( + !(await rewriteMemoryHostEventArtifactIfUnchanged({ + workspaceRoot, + relativePath: owner.ownerRelativePath, + expectedContent: ownership.ownerContent, + nextContent: updateOwnerContent, + })) + ) { + return undefined; + } + await syncDirectoryBestEffort(path.dirname(absolutePath)); + if ( + !(await rewriteMemoryHostEventArtifactIfUnchanged({ + workspaceRoot, + relativePath: owner.relativePath, + expectedIdentity: ownership.identity, + nextContent: content, + })) + ) { + return undefined; + } + await syncDirectoryBestEffort(path.dirname(absolutePath)); + if ( + !(await rewriteMemoryHostEventArtifactIfUnchanged({ + workspaceRoot, + relativePath: owner.ownerRelativePath, + expectedContent: updateOwnerContent, + nextContent: currentOwnerContent, + })) + ) { + return undefined; + } + await syncDirectoryBestEffort(path.dirname(absolutePath)); + } catch (error) { + if (isWorkspaceWriteUnavailable(error)) { + return undefined; + } + throw error; + } + } else if (ownership.needsFinalize) { + publishedIdentity = ownership.identity; + try { + if ( + !(await rewriteMemoryHostEventArtifactIfUnchanged({ + workspaceRoot, + relativePath: owner.ownerRelativePath, + expectedContent: ownership.ownerContent, + nextContent: memoryHostEventExportOwnerContent(owner, { + currentSha256: contentSha256, + identity: ownership.identity, + }), + })) + ) { + return undefined; + } + await syncDirectoryBestEffort(path.dirname(absolutePath)); + } catch (error) { + if (isWorkspaceWriteUnavailable(error)) { + return undefined; + } + throw error; + } + } else { + publishedIdentity = ownership.identity; + } + if (storedEvents.length === 0 || !publishedIdentity) { + return undefined; + } + return (await isMemoryHostEventArtifactAtIdentity({ + workspaceRoot, + relativePath: owner.relativePath, + expectedIdentity: publishedIdentity, + expectedContent: content, + })) + ? { absolutePath, relativePath: owner.relativePath } + : undefined; + }); + }); +} + /** Lists public memory artifacts for one workspace, including notes and event logs. */ async function listMemoryWorkspacePublicArtifacts(params: { workspaceDir: string; @@ -87,13 +486,15 @@ async function listMemoryWorkspacePublicArtifacts(params: { }); } - const eventLogPath = resolveMemoryHostEventLogPath(params.workspaceDir); - if (await pathExists(eventLogPath)) { + const eventExport = await materializeMemoryHostEventExport({ + workspaceDir: params.workspaceDir, + }); + if (eventExport) { artifacts.push({ kind: "event-log", workspaceDir: params.workspaceDir, - relativePath: path.relative(params.workspaceDir, eventLogPath).replace(/\\/g, "/"), - absolutePath: eventLogPath, + relativePath: eventExport.relativePath, + absolutePath: eventExport.absolutePath, agentIds: [...params.agentIds], contentType: "json", }); diff --git a/src/plugin-sdk/memory-host-event-export.test.ts b/src/plugin-sdk/memory-host-event-export.test.ts new file mode 100644 index 00000000000..26914323f4e --- /dev/null +++ b/src/plugin-sdk/memory-host-event-export.test.ts @@ -0,0 +1,223 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { root as createFsSafeRoot } from "../infra/fs-safe.js"; +import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; +import { clearMemoryPluginState } from "../plugins/memory-state.test-fixtures.js"; +import { listMemoryHostPublicArtifacts } from "./memory-host-core.js"; +import { + memoryHostEventExportOwnerContent, + publishMemoryHostEventArtifact, +} from "./memory-host-event-export.js"; +import { appendMemoryHostEvent } from "./memory-host-events.js"; + +describe("memory host event export recovery", () => { + afterEach(() => { + clearMemoryPluginState(); + resetPluginStateStoreForTests(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("does not finalize an initial export changed through the published inode", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-publish-race-")); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const relativePath = "memory/events/state/memory-host-events.jsonl"; + const ownerRelativePath = "memory/events/state/.openclaw-memory-host-events-owner.json"; + const absolutePath = path.join(workspaceDir, relativePath); + const owner = { + queueKey: "state\0workspace", + lockTarget: path.join(fixtureRoot, "lock"), + relativePath, + ownerRelativePath, + stateHash: "state", + workspaceHash: "workspace", + }; + const content = '{"type":"memory.recall.recorded","query":"�"}\n'; + const expectedBytes = Buffer.from(content, "utf8"); + const replacementBytes = Buffer.from("�", "utf8"); + const replacementOffset = expectedBytes.indexOf(replacementBytes); + const foreignBytes = Buffer.concat([ + expectedBytes.subarray(0, replacementOffset), + Buffer.from([0x80]), + expectedBytes.subarray(replacementOffset + replacementBytes.length), + ]); + const contentSha256 = createHash("sha256").update(content).digest("hex"); + const expectedOwnerContent = memoryHostEventExportOwnerContent(owner, { + pendingSha256: contentSha256, + }); + try { + await fs.mkdir(path.dirname(absolutePath), { recursive: true }); + await fs.writeFile(path.join(workspaceDir, ownerRelativePath), expectedOwnerContent, "utf8"); + const workspaceRoot = await createFsSafeRoot(workspaceDir, { + hardlinks: "reject", + mkdir: true, + mode: 0o600, + symlinks: "reject", + }); + const originalOpen = workspaceRoot.open.bind(workspaceRoot); + let replacedContent = false; + const interceptedWorkspaceRoot = new Proxy(workspaceRoot, { + get(target, property, receiver) { + if (property === "open") { + return async ( + openedRelativePath: string, + options?: Parameters[1], + ) => { + if (openedRelativePath === relativePath && !replacedContent) { + replacedContent = true; + await fs.writeFile(absolutePath, foreignBytes); + } + return await originalOpen(openedRelativePath, options); + }; + } + const value = Reflect.get(target, property, receiver) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + + await expect( + publishMemoryHostEventArtifact({ + workspaceRoot: interceptedWorkspaceRoot, + owner, + absolutePath, + expectedOwnerContent, + content, + contentSha256, + }), + ).resolves.toBeUndefined(); + expect(foreignBytes.toString("utf8")).toBe(content); + await expect(fs.readFile(absolutePath)).resolves.toEqual(foreignBytes); + expect( + JSON.parse(await fs.readFile(path.join(workspaceDir, ownerRelativePath), "utf8")), + ).toMatchObject({ + pendingContentSha256: contentSha256, + fileDev: expect.any(String), + fileIno: expect.any(String), + }); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("finishes an inode-owned empty event export after interruption", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-inode-owner-")); + const stateDir = path.join(fixtureRoot, "state"); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const event = { + type: "memory.recall.recorded" as const, + timestamp: "2026-05-18T12:00:00.000Z", + query: "recover inode-owned export", + resultCount: 0, + results: [], + }; + try { + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + await fs.mkdir(workspaceDir); + await appendMemoryHostEvent(workspaceDir, event); + const stateHash = createHash("sha256") + .update(await fs.realpath(stateDir)) + .digest("hex") + .slice(0, 32); + const workspaceHash = createHash("sha256") + .update(await fs.realpath(workspaceDir)) + .digest("hex") + .slice(0, 32); + const exportDir = path.join(workspaceDir, "memory", "events", stateHash); + const exportPath = path.join(exportDir, "memory-host-events.jsonl"); + const ownerPath = path.join(exportDir, ".openclaw-memory-host-events-owner.json"); + const expectedContent = `${JSON.stringify(event)}\n`; + await fs.mkdir(exportDir, { recursive: true }); + await fs.writeFile(exportPath, "", { mode: 0o600 }); + const exportStat = await fs.stat(exportPath, { bigint: true }); + await fs.writeFile( + ownerPath, + `${JSON.stringify({ + schemaVersion: 3, + kind: "openclaw-memory-host-events-export", + stateHash, + workspaceHash, + pendingContentSha256: createHash("sha256").update(expectedContent).digest("hex"), + fileDev: String(exportStat.dev), + fileIno: String(exportStat.ino), + })}\n`, + "utf8", + ); + + const listed = await listMemoryHostPublicArtifacts({ + cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } }, + }); + + expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(true); + await expect(fs.readFile(exportPath, "utf8")).resolves.toBe(expectedContent); + const owner = JSON.parse(await fs.readFile(ownerPath, "utf8")) as { + contentSha256?: string; + fileDev?: string; + fileIno?: string; + pendingContentSha256?: string; + }; + expect(owner).toMatchObject({ + contentSha256: createHash("sha256").update(expectedContent).digest("hex"), + fileDev: expect.any(String), + fileIno: expect.any(String), + }); + expect(owner.pendingContentSha256).toBeUndefined(); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("does not claim an empty export after exclusive-create interruption", async () => { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-empty-export-")); + const stateDir = path.join(fixtureRoot, "state"); + const workspaceDir = path.join(fixtureRoot, "workspace"); + const event = { + type: "memory.recall.recorded" as const, + timestamp: "2026-05-18T12:00:00.000Z", + query: "leave empty export untouched", + resultCount: 0, + results: [], + }; + try { + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + await fs.mkdir(workspaceDir); + await appendMemoryHostEvent(workspaceDir, event); + const stateHash = createHash("sha256") + .update(await fs.realpath(stateDir)) + .digest("hex") + .slice(0, 32); + const workspaceHash = createHash("sha256") + .update(await fs.realpath(workspaceDir)) + .digest("hex") + .slice(0, 32); + const exportDir = path.join(workspaceDir, "memory", "events", stateHash); + const exportPath = path.join(exportDir, "memory-host-events.jsonl"); + const ownerPath = path.join(exportDir, ".openclaw-memory-host-events-owner.json"); + const expectedContent = `${JSON.stringify(event)}\n`; + await fs.mkdir(exportDir, { recursive: true }); + await fs.writeFile(exportPath, "", { mode: 0o600 }); + await fs.writeFile( + ownerPath, + `${JSON.stringify({ + schemaVersion: 3, + kind: "openclaw-memory-host-events-export", + stateHash, + workspaceHash, + pendingContentSha256: createHash("sha256").update(expectedContent).digest("hex"), + })}\n`, + "utf8", + ); + + const listed = await listMemoryHostPublicArtifacts({ + cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } }, + }); + + expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(false); + await expect(fs.readFile(exportPath, "utf8")).resolves.toBe(""); + } finally { + await fs.rm(fixtureRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/src/plugin-sdk/memory-host-event-export.ts b/src/plugin-sdk/memory-host-event-export.ts new file mode 100644 index 00000000000..5636ed9f847 --- /dev/null +++ b/src/plugin-sdk/memory-host-event-export.ts @@ -0,0 +1,268 @@ +import type { FileHandle } from "node:fs/promises"; +import path from "node:path"; +import { sameFileIdentity, type FileIdentityStat } from "../infra/fs-safe-advanced.js"; +import { FsSafeError, root as createFsSafeRoot } from "../infra/fs-safe.js"; +import { syncDirectoryBestEffort } from "../infra/sqlite-snapshot.js"; + +export type MemoryHostEventExportOwner = { + queueKey: string; + lockTarget: string; + relativePath: string; + ownerRelativePath: string; + stateHash: string; + workspaceHash: string; +}; + +type MemoryHostWorkspaceRoot = Awaited>; + +export function isMissingPathError(error: unknown): boolean { + const code = (error as { code?: unknown }).code; + return ( + code === "ENOENT" || + code === "ENOTDIR" || + (error instanceof FsSafeError && code === "not-found") + ); +} + +export function isRejectedWorkspaceArtifactPath(error: unknown): boolean { + if (!(error instanceof FsSafeError)) { + return false; + } + return ( + error.code === "hardlink" || + error.code === "not-file" || + error.code === "outside-workspace" || + error.code === "path-alias" || + error.code === "path-mismatch" || + error.code === "symlink" + ); +} + +export function memoryHostEventExportOwnerContent( + owner: MemoryHostEventExportOwner, + content: { + currentSha256?: string; + pendingSha256?: string; + identity?: FileIdentityStat; + }, +): string { + return `${JSON.stringify({ + schemaVersion: 3, + kind: "openclaw-memory-host-events-export", + stateHash: owner.stateHash, + workspaceHash: owner.workspaceHash, + ...(content.identity + ? { fileDev: String(content.identity.dev), fileIno: String(content.identity.ino) } + : {}), + ...(content.currentSha256 ? { contentSha256: content.currentSha256 } : {}), + ...(content.pendingSha256 ? { pendingContentSha256: content.pendingSha256 } : {}), + })}\n`; +} + +async function writePinnedMemoryHostEventArtifact( + handle: FileHandle, + content: string, +): Promise { + const bytes = Buffer.from(content, "utf8"); + let offset = 0; + while (offset < bytes.length) { + const result = await handle.write(bytes, offset, bytes.length - offset, offset); + if (result.bytesWritten === 0) { + throw new Error("event export write made no progress"); + } + offset += result.bytesWritten; + } + await handle.truncate(bytes.length); + await handle.chmod(0o600); + await handle.sync(); +} + +export async function rewriteMemoryHostEventArtifactIfUnchanged(params: { + workspaceRoot: MemoryHostWorkspaceRoot; + relativePath: string; + expectedContent?: string; + expectedIdentity?: FileIdentityStat; + nextContent: string; +}): Promise { + let observed: Awaited>; + try { + observed = await params.workspaceRoot.open(params.relativePath); + } catch (error) { + if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) { + return false; + } + throw error; + } + try { + if ( + params.expectedIdentity + ? !sameFileIdentity(params.expectedIdentity, observed.stat) + : (await observed.handle.readFile({ encoding: "utf8" })) !== params.expectedContent + ) { + return false; + } + let writable: Awaited>; + try { + writable = await params.workspaceRoot.openWritable(params.relativePath, { + mode: 0o600, + writeMode: "update", + }); + } catch (error) { + if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) { + return false; + } + throw error; + } + try { + // The matching marker/content snapshot owns this generated artifact inode for + // the locked update. A distinct workspace file can take the path only by + // replacing that inode; direct writes still target the owned export itself. + if (!sameFileIdentity(observed.stat, writable.stat)) { + return false; + } + await writable.handle.writeFile(params.nextContent, { encoding: "utf8" }); + await writable.handle.truncate(Buffer.byteLength(params.nextContent, "utf8")); + await writable.handle.chmod(0o600); + await writable.handle.sync(); + } finally { + await writable.handle.close().catch(() => undefined); + } + let verified: Awaited>; + try { + verified = await params.workspaceRoot.open(params.relativePath); + } catch (error) { + if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) { + return false; + } + throw error; + } + try { + return ( + sameFileIdentity(observed.stat, verified.stat) && + (await verified.handle.readFile({ encoding: "utf8" })) === params.nextContent + ); + } finally { + await verified.handle.close().catch(() => undefined); + } + } finally { + await observed.handle.close().catch(() => undefined); + } +} + +export async function isMemoryHostEventArtifactAtIdentity(params: { + workspaceRoot: MemoryHostWorkspaceRoot; + relativePath: string; + expectedIdentity: FileIdentityStat; + expectedContent?: string; +}): Promise { + let opened: Awaited>; + try { + opened = await params.workspaceRoot.open(params.relativePath); + } catch (error) { + if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) { + return false; + } + throw error; + } + try { + if (!sameFileIdentity(params.expectedIdentity, opened.stat)) { + return false; + } + return ( + params.expectedContent === undefined || + (await opened.handle.readFile()).equals(Buffer.from(params.expectedContent, "utf8")) + ); + } finally { + await opened.handle.close().catch(() => undefined); + } +} + +export async function publishMemoryHostEventArtifact(params: { + workspaceRoot: MemoryHostWorkspaceRoot; + owner: MemoryHostEventExportOwner; + absolutePath: string; + expectedOwnerContent: string; + content: string; + contentSha256: string; +}): Promise { + let writable: Awaited>; + try { + writable = await params.workspaceRoot.openWritable(params.owner.relativePath, { + mode: 0o600, + writeMode: "replace", + }); + } catch (error) { + if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) { + return undefined; + } + throw error; + } + try { + // `createdForWrite` is the exclusive-create proof. Keep its handle pinned + // through publication so a replacement path is never opened for mutation. + if (!writable.createdForWrite) { + return undefined; + } + const publishedIdentity = { dev: writable.stat.dev, ino: writable.stat.ino }; + await syncDirectoryBestEffort(path.dirname(params.absolutePath)); + + const identityPendingOwnerContent = memoryHostEventExportOwnerContent(params.owner, { + pendingSha256: params.contentSha256, + identity: publishedIdentity, + }); + if ( + !(await rewriteMemoryHostEventArtifactIfUnchanged({ + workspaceRoot: params.workspaceRoot, + relativePath: params.owner.ownerRelativePath, + expectedContent: params.expectedOwnerContent, + nextContent: identityPendingOwnerContent, + })) + ) { + return undefined; + } + await syncDirectoryBestEffort(path.dirname(params.absolutePath)); + + await writePinnedMemoryHostEventArtifact(writable.handle, params.content); + // Workspace actors can mutate this inode without replacing the path. Verify + // bytes before finalizing the marker so foreign content never gains ownership. + if ( + !(await isMemoryHostEventArtifactAtIdentity({ + workspaceRoot: params.workspaceRoot, + relativePath: params.owner.relativePath, + expectedIdentity: publishedIdentity, + expectedContent: params.content, + })) + ) { + return undefined; + } + await syncDirectoryBestEffort(path.dirname(params.absolutePath)); + + if ( + !(await rewriteMemoryHostEventArtifactIfUnchanged({ + workspaceRoot: params.workspaceRoot, + relativePath: params.owner.ownerRelativePath, + expectedContent: identityPendingOwnerContent, + nextContent: memoryHostEventExportOwnerContent(params.owner, { + currentSha256: params.contentSha256, + identity: publishedIdentity, + }), + })) + ) { + return undefined; + } + await syncDirectoryBestEffort(path.dirname(params.absolutePath)); + if ( + !(await isMemoryHostEventArtifactAtIdentity({ + workspaceRoot: params.workspaceRoot, + relativePath: params.owner.relativePath, + expectedIdentity: publishedIdentity, + expectedContent: params.content, + })) + ) { + return undefined; + } + return publishedIdentity; + } finally { + await writable.handle.close().catch(() => undefined); + } +} diff --git a/src/plugin-sdk/memory-host-events.test.ts b/src/plugin-sdk/memory-host-events.test.ts index 3f2dad57adb..ec35aaeeb20 100644 --- a/src/plugin-sdk/memory-host-events.test.ts +++ b/src/plugin-sdk/memory-host-events.test.ts @@ -3,13 +3,16 @@ */ import fs from "node:fs/promises"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + listStoredMemoryHostEvents, + setMaxMemoryHostEventsForTests, +} from "../memory-host-sdk/event-store.js"; import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; import { appendMemoryHostEvent, readMemoryHostEventRecords, readMemoryHostEvents, - resolveMemoryHostEventLogPath, } from "./memory-host-events.js"; import { createClaimableDedupe, @@ -32,45 +35,51 @@ function createDedupe(root: string, overrides?: { ttlMs?: number }) { } afterEach(() => { + vi.restoreAllMocks(); + setMaxMemoryHostEventsForTests(undefined); resetPluginStateStoreForTests(); }); describe("memory host event journal helpers", () => { it("appends and reads typed workspace events", async () => { const workspaceDir = await createTempDir("memory-host-events-"); + const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir }; - await appendMemoryHostEvent(workspaceDir, { - type: "memory.recall.recorded", - timestamp: "2026-04-05T12:00:00.000Z", - query: "glacier backup", - resultCount: 1, - results: [ - { - path: "memory/2026-04-05.md", - startLine: 1, - endLine: 3, - score: 0.9, - }, - ], - }); - await appendMemoryHostEvent(workspaceDir, { - type: "memory.dream.completed", - timestamp: "2026-04-05T13:00:00.000Z", - phase: "light", - outcome: "completed", - lineCount: 4, - storageMode: "both", - inlinePath: path.join(workspaceDir, "memory", "2026-04-05.md"), - reportPath: path.join(workspaceDir, "memory", "dreaming", "light", "2026-04-05.md"), - }); - - const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir); - await expect(fs.readFile(eventLogPath, "utf8")).resolves.toContain( - '"type":"memory.recall.recorded"', + await appendMemoryHostEvent( + workspaceDir, + { + type: "memory.recall.recorded", + timestamp: "2026-04-05T12:00:00.000Z", + query: "glacier backup", + resultCount: 1, + results: [ + { + path: "memory/2026-04-05.md", + startLine: 1, + endLine: 3, + score: 0.9, + }, + ], + }, + { env }, + ); + await appendMemoryHostEvent( + workspaceDir, + { + type: "memory.dream.completed", + timestamp: "2026-04-05T11:00:00.000Z", + phase: "light", + outcome: "completed", + lineCount: 4, + storageMode: "both", + inlinePath: path.join(workspaceDir, "memory", "2026-04-05.md"), + reportPath: path.join(workspaceDir, "memory", "dreaming", "light", "2026-04-05.md"), + }, + { env }, ); - const events = await readMemoryHostEvents({ workspaceDir }); - const tail = await readMemoryHostEvents({ workspaceDir, limit: 1 }); + const events = await readMemoryHostEvents({ workspaceDir, env }); + const tail = await readMemoryHostEvents({ workspaceDir, limit: 1, env }); expect(events).toHaveLength(2); expect(events[0]?.type).toBe("memory.recall.recorded"); @@ -83,62 +92,101 @@ describe("memory host event journal helpers", () => { expect(tail[0]?.type).toBe("memory.dream.completed"); }); - it("keeps legacy event readers stable when diagnostic records are present", async () => { - const workspaceDir = await createTempDir("memory-host-events-diagnostics-"); + it("keeps journal retention timestamps in the current wall-clock domain", async () => { + const workspaceDir = await createTempDir("memory-host-events-created-at-"); + const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir }; + const now = Date.parse("2026-07-16T12:00:00.000Z"); + vi.useFakeTimers(); + vi.setSystemTime(now); - await appendMemoryHostEvent(workspaceDir, { - type: "memory.recall.skipped", - timestamp: "2026-04-05T12:00:00.000Z", - query: "durable memory", - reason: "non-short-term-memory-path", - eligibleResultCount: 0, - skippedResultCount: 1, - results: [ + for (const query of ["first", "second"]) { + await appendMemoryHostEvent( + workspaceDir, { - path: "MEMORY.md", - startLine: 3, - endLine: 3, - score: 0.9, - reason: "non-short-term-memory-path", + type: "memory.recall.recorded", + timestamp: "2026-07-16T12:00:00.000Z", + query, + resultCount: 0, + results: [], }, - ], - }); + { env }, + ); + } - await appendMemoryHostEvent(workspaceDir, { - type: "memory.recall.recorded", - timestamp: "2026-04-05T12:05:00.000Z", - query: "daily memory", - resultCount: 1, - results: [ - { - path: "memory/2026-04-05.md", - startLine: 1, - endLine: 3, - score: 0.95, - }, - ], - }); - await appendMemoryHostEvent(workspaceDir, { - type: "memory.recall.skipped", - timestamp: "2026-04-05T12:10:00.000Z", - query: "durable memory again", - reason: "non-short-term-memory-path", - eligibleResultCount: 1, - skippedResultCount: 1, - results: [ - { - path: "MEMORY.md", - startLine: 4, - endLine: 4, - score: 0.8, - reason: "non-short-term-memory-path", - }, - ], - }); + expect( + listStoredMemoryHostEvents({ workspaceDir, env }).map((entry) => entry.createdAt), + ).toEqual([now, now + 1]); + }); - const legacyEvents = await readMemoryHostEvents({ workspaceDir }); - const legacyTail = await readMemoryHostEvents({ workspaceDir, limit: 1 }); - const records = await readMemoryHostEventRecords({ workspaceDir }); + it("keeps legacy event readers stable when diagnostic records are present", async () => { + const workspaceDir = await createTempDir("memory-host-events-diagnostics-"); + const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir }; + + await appendMemoryHostEvent( + workspaceDir, + { + type: "memory.recall.skipped", + timestamp: "2026-04-05T12:00:00.000Z", + query: "durable memory", + reason: "non-short-term-memory-path", + eligibleResultCount: 0, + skippedResultCount: 1, + results: [ + { + path: "MEMORY.md", + startLine: 3, + endLine: 3, + score: 0.9, + reason: "non-short-term-memory-path", + }, + ], + }, + { env }, + ); + + await appendMemoryHostEvent( + workspaceDir, + { + type: "memory.recall.recorded", + timestamp: "2026-04-05T12:05:00.000Z", + query: "daily memory", + resultCount: 1, + results: [ + { + path: "memory/2026-04-05.md", + startLine: 1, + endLine: 3, + score: 0.95, + }, + ], + }, + { env }, + ); + await appendMemoryHostEvent( + workspaceDir, + { + type: "memory.recall.skipped", + timestamp: "2026-04-05T12:10:00.000Z", + query: "durable memory again", + reason: "non-short-term-memory-path", + eligibleResultCount: 1, + skippedResultCount: 1, + results: [ + { + path: "MEMORY.md", + startLine: 4, + endLine: 4, + score: 0.8, + reason: "non-short-term-memory-path", + }, + ], + }, + { env }, + ); + + const legacyEvents = await readMemoryHostEvents({ workspaceDir, env }); + const legacyTail = await readMemoryHostEvents({ workspaceDir, limit: 1, env }); + const records = await readMemoryHostEventRecords({ workspaceDir, env }); expect(legacyEvents.map((event) => event.type)).toEqual(["memory.recall.recorded"]); expect(legacyTail.map((event) => event.type)).toEqual(["memory.recall.recorded"]); @@ -148,6 +196,105 @@ describe("memory host event journal helpers", () => { "memory.recall.skipped", ]); }); + + it("bounds oversized diagnostic detail without failing the parent operation", async () => { + const workspaceDir = await createTempDir("memory-host-events-bounded-"); + const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir }; + const results = Array.from({ length: 100 }, (_, index) => ({ + path: `memory/${"wide-path-".repeat(100)}${index}.md`, + startLine: index + 1, + endLine: index + 2, + score: 0.9, + })); + + await expect( + appendMemoryHostEvent( + workspaceDir, + { + type: "memory.recall.recorded", + timestamp: "2026-04-05T12:00:00.000Z", + query: "🔥".repeat(20_000), + resultCount: results.length, + results, + }, + { env }, + ), + ).resolves.toBeUndefined(); + + const [event] = await readMemoryHostEventRecords({ workspaceDir, env }); + expect(event).toMatchObject({ + type: "memory.recall.recorded", + resultCount: 100, + storageTruncated: true, + }); + if (event?.type !== "memory.recall.recorded") { + throw new Error("expected bounded recall event"); + } + expect(event.results).toHaveLength(10); + expect(Buffer.byteLength(JSON.stringify(event), "utf8")).toBeLessThanOrEqual(8 * 1024); + }); + + it("rotates old events without evicting the workspace sequence cursor", async () => { + const workspaceDir = await createTempDir("memory-host-events-rotation-"); + const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir }; + setMaxMemoryHostEventsForTests(3); + let clock = 1_000; + vi.spyOn(Date, "now").mockImplementation(() => clock--); + + for (let index = 1; index <= 5; index += 1) { + await appendMemoryHostEvent( + workspaceDir, + { + type: "memory.recall.recorded", + timestamp: `2026-04-05T12:00:0${index}.000Z`, + query: `event-${index}`, + resultCount: 0, + results: [], + }, + { env }, + ); + } + + const events = await readMemoryHostEventRecords({ workspaceDir, env }); + expect( + events.map((event) => (event.type === "memory.recall.recorded" ? event.query : "")), + ).toEqual(["event-3", "event-4", "event-5"]); + }); + + it("rotates events by namespace append order across workspaces", async () => { + const stateDir = await createTempDir("memory-host-events-shared-retention-"); + const workspaceA = path.join(stateDir, "workspace-a"); + const workspaceB = path.join(stateDir, "workspace-b"); + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + setMaxMemoryHostEventsForTests(3); + + const appendRecall = async (workspaceDir: string, query: string) => { + await appendMemoryHostEvent( + workspaceDir, + { + type: "memory.recall.recorded", + timestamp: "2026-04-05T12:00:00.000Z", + query, + resultCount: 0, + results: [], + }, + { env }, + ); + }; + await appendRecall(workspaceA, "a-1"); + await appendRecall(workspaceA, "a-2"); + await appendRecall(workspaceA, "a-3"); + await appendRecall(workspaceB, "b-1"); + await appendRecall(workspaceA, "a-4"); + + const workspaceBEvents = await readMemoryHostEventRecords({ + workspaceDir: workspaceB, + env, + }); + expect( + workspaceBEvents.map((event) => (event.type === "memory.recall.recorded" ? event.query : "")), + ).toEqual(["b-1"]); + }); }); describe("createPersistentDedupe", () => { diff --git a/src/plugin-sdk/memory-host-events.ts b/src/plugin-sdk/memory-host-events.ts index 09cccc28d2c..8cda8e2299b 100644 --- a/src/plugin-sdk/memory-host-events.ts +++ b/src/plugin-sdk/memory-host-events.ts @@ -1,8 +1,9 @@ /** - * Public SDK subpath for memory host event log types and helpers. + * Public SDK subpath for memory host event types and helpers. */ export { appendMemoryHostEvent, + normalizeMemoryHostEventRecordForStorage, readMemoryHostEventRecords, readMemoryHostEvents, resolveMemoryHostEventLogPath, diff --git a/src/plugin-sdk/plugin-state-test-runtime.ts b/src/plugin-sdk/plugin-state-test-runtime.ts index fb3750b473e..69949122f6c 100644 --- a/src/plugin-sdk/plugin-state-test-runtime.ts +++ b/src/plugin-sdk/plugin-state-test-runtime.ts @@ -4,8 +4,11 @@ export { createPluginStateKeyedStore as createPluginStateKeyedStoreForTests, createPluginStateSyncKeyedStore as createPluginStateSyncKeyedStoreForTests, + getPluginStateCapacity as getPluginStateCapacityForTests, + importPluginStateEntriesForDoctor as importPluginStateEntriesForDoctorForTests, resetPluginStateStoreForTests, } from "../plugin-state/plugin-state-store.js"; +export { setMaxMemoryHostEventsForTests } from "../memory-host-sdk/event-store.js"; export { createPluginBlobStoreForTests, resetPluginBlobStoreForTests, diff --git a/src/plugin-state/plugin-state-store.sqlite.ts b/src/plugin-state/plugin-state-store.sqlite.ts index 6abc312c8b1..b078b4a54b7 100644 --- a/src/plugin-state/plugin-state-store.sqlite.ts +++ b/src/plugin-state/plugin-state-store.sqlite.ts @@ -222,6 +222,33 @@ function selectPluginStateEntries( ).rows; } +function selectPluginStateEntriesInKeyRange( + db: DatabaseSync, + params: { + pluginId: string; + namespace: string; + keyStartInclusive: string; + keyEndExclusive: string; + limit: number; + order: "asc" | "desc"; + now: number; + }, +): PluginStateRow[] { + return executeSqliteQuerySync( + db, + getPluginStateKysely(db) + .selectFrom("plugin_state_entries") + .select(["plugin_id", "namespace", "entry_key", "value_json", "created_at", "expires_at"]) + .where("plugin_id", "=", params.pluginId) + .where("namespace", "=", params.namespace) + .where("entry_key", ">=", params.keyStartInclusive) + .where("entry_key", "<", params.keyEndExclusive) + .where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", params.now)])) + .orderBy("entry_key", params.order) + .limit(params.limit), + ).rows; +} + function deletePluginStateEntry( db: DatabaseSync, params: { pluginId: string; namespace: string; key: string }, @@ -268,6 +295,26 @@ function countLivePluginStateNamespaceEntries( return countRow(row); } +function allocatePluginStateNamespaceCreatedAt( + db: DatabaseSync, + params: { pluginId: string; namespace: string; now: number }, +): number { + const row = executeSqliteQueryTakeFirstSync( + db, + getPluginStateKysely(db) + .selectFrom("plugin_state_entries") + .select((eb) => eb.fn.max("created_at").as("max_created_at")) + .where("plugin_id", "=", params.pluginId) + .where("namespace", "=", params.namespace), + ); + const previous = normalizeSqliteNumber(row?.max_created_at ?? null); + const next = previous === undefined ? params.now : Math.max(params.now, previous + 1); + if (!Number.isSafeInteger(next)) { + throw new RangeError("Plugin state namespace append order exhausted safe integer range"); + } + return next; +} + function countLivePluginStateEntries( db: DatabaseSync, params: { pluginId: string; now: number }, @@ -380,6 +427,7 @@ function enforcePostRegisterLimits(params: { overflowPolicy: PluginStateOverflowPolicy; now: number; protectedKey: string; + enforcePluginLimit?: boolean; }): void { if (params.overflowPolicy === "reject-new") { return; @@ -399,6 +447,10 @@ function enforcePostRegisterLimits(params: { }); } + if (params.enforcePluginLimit === false) { + return; + } + const pluginCount = countLivePluginStateEntries(params.store.db, { pluginId: params.pluginId, now: params.now, @@ -408,7 +460,9 @@ function enforcePostRegisterLimits(params: { return; } - // Shed rows from the namespace that grew before failing the plugin write. + // Shed only rows from the namespace that grew. Sibling namespaces can hold + // durable state; if this namespace cannot cover the overflow, fail so the + // surrounding transaction rolls every insertion and deletion back. deleteOldestPluginStateNamespaceEntries(params.store.db, { pluginId: params.pluginId, namespace: params.namespace, @@ -551,6 +605,141 @@ export function pluginStateRegister(params: { } } +export function pluginStateRegisterSequencedJournalEntry(params: { + pluginId: string; + cursorNamespace: string; + cursorKey: string; + cursorMaxEntries: number; + journalNamespace: string; + journalMaxEntries: number; + initialSequence: number; + readCursorSequence: (valueJson: string) => number | undefined; + prepareEntry: (sequence: number) => { + cursorValueJson: string; + journalKey: string; + journalValueJson: string; + }; + env?: NodeJS.ProcessEnv; +}): number { + try { + return runWriteTransaction( + "register", + (store) => { + const now = Date.now(); + deleteExpiredPluginStateNamespaceEntries(store.db, { + pluginId: params.pluginId, + namespace: params.cursorNamespace, + now, + }); + deleteExpiredPluginStateNamespaceEntries(store.db, { + pluginId: params.pluginId, + namespace: params.journalNamespace, + now, + }); + const cursor = selectPluginStateEntry(store.db, { + pluginId: params.pluginId, + namespace: params.cursorNamespace, + key: params.cursorKey, + now, + }); + const cursorSequence = cursor ? params.readCursorSequence(cursor.value_json) : undefined; + const lastSequence = Math.max(params.initialSequence, cursorSequence ?? 0); + const sequence = lastSequence + 1; + if (!Number.isSafeInteger(sequence)) { + throw new RangeError("Plugin state journal sequence exhausted safe integer range"); + } + const prepared = params.prepareEntry(sequence); + const existingJournalEntry = selectPluginStateEntry(store.db, { + pluginId: params.pluginId, + namespace: params.journalNamespace, + key: prepared.journalKey, + now, + }); + if (existingJournalEntry) { + throw createPluginStateError({ + code: "PLUGIN_STATE_WRITE_FAILED", + operation: "register", + message: "Plugin state journal sequence already exists.", + path: store.path, + }); + } + if (!cursor) { + assertCanInsertPluginStateEntry({ + store, + pluginId: params.pluginId, + namespace: params.cursorNamespace, + maxEntries: params.cursorMaxEntries, + overflowPolicy: "evict-oldest", + now, + }); + } + assertCanInsertPluginStateEntry({ + store, + pluginId: params.pluginId, + namespace: params.journalNamespace, + maxEntries: params.journalMaxEntries, + overflowPolicy: "evict-oldest", + now, + }); + upsertPluginStateEntry( + store.db, + bindPluginStateEntry({ + pluginId: params.pluginId, + namespace: params.cursorNamespace, + key: params.cursorKey, + valueJson: prepared.cursorValueJson, + createdAt: now, + expiresAt: null, + }), + ); + enforcePostRegisterLimits({ + store, + pluginId: params.pluginId, + namespace: params.cursorNamespace, + maxEntries: params.cursorMaxEntries, + overflowPolicy: "evict-oldest", + now, + protectedKey: params.cursorKey, + enforcePluginLimit: false, + }); + upsertPluginStateEntry( + store.db, + bindPluginStateEntry({ + pluginId: params.pluginId, + namespace: params.journalNamespace, + key: prepared.journalKey, + valueJson: prepared.journalValueJson, + createdAt: allocatePluginStateNamespaceCreatedAt(store.db, { + pluginId: params.pluginId, + namespace: params.journalNamespace, + now, + }), + expiresAt: null, + }), + ); + enforcePostRegisterLimits({ + store, + pluginId: params.pluginId, + namespace: params.journalNamespace, + maxEntries: params.journalMaxEntries, + overflowPolicy: "evict-oldest", + now, + protectedKey: prepared.journalKey, + }); + return sequence; + }, + envOptions(params.env), + ); + } catch (error) { + throw wrapPluginStateError( + error, + "register", + "PLUGIN_STATE_WRITE_FAILED", + "Failed to register sequenced plugin state journal entry.", + ); + } +} + export function pluginStateRegisterIfAbsent(params: { pluginId: string; namespace: string; @@ -852,6 +1041,51 @@ export function pluginStateEntries(params: { } } +/** Internal bounded key-range read for core owners with sortable plugin-state keys. */ +export function pluginStateEntriesInKeyRange(params: { + pluginId: string; + namespace: string; + keyStartInclusive: string; + keyEndExclusive: string; + limit: number; + order?: "asc" | "desc"; + env?: NodeJS.ProcessEnv; +}): PluginStateEntry[] { + if (!Number.isSafeInteger(params.limit) || params.limit < 1) { + throw createPluginStateError({ + code: "PLUGIN_STATE_INVALID_INPUT", + operation: "entries", + message: "Plugin state key-range limit must be a positive safe integer.", + }); + } + if (params.keyStartInclusive >= params.keyEndExclusive) { + throw createPluginStateError({ + code: "PLUGIN_STATE_INVALID_INPUT", + operation: "entries", + message: "Plugin state key range must have an increasing exclusive upper bound.", + }); + } + try { + const { db } = openPluginStateDatabase("entries", envOptions(params.env)); + return selectPluginStateEntriesInKeyRange(db, { + pluginId: params.pluginId, + namespace: params.namespace, + keyStartInclusive: params.keyStartInclusive, + keyEndExclusive: params.keyEndExclusive, + limit: params.limit, + order: params.order ?? "asc", + now: Date.now(), + }).map((row) => rowToEntry(row, "entries")); + } catch (error) { + throw wrapPluginStateError( + error, + "entries", + "PLUGIN_STATE_READ_FAILED", + "Failed to list plugin state entries by key range.", + ); + } +} + export function pluginStateClear(params: { pluginId: string; namespace: string; @@ -912,9 +1146,9 @@ function setMaxPluginStateEntriesPerPluginForTests(value?: number): void { maxPluginStateEntriesPerPluginForTests = value; } -export function countPluginStateLiveEntries(pluginId: string): number { +export function countPluginStateLiveEntries(pluginId: string, env?: NodeJS.ProcessEnv): number { try { - const { db } = openPluginStateDatabase("entries"); + const { db } = openPluginStateDatabase("entries", envOptions(env)); return countLivePluginStateEntries(db, { pluginId, now: Date.now() }); } catch (error) { throw wrapPluginStateError( @@ -926,6 +1160,16 @@ export function countPluginStateLiveEntries(pluginId: string): number { } } +export function getPluginStateCapacity( + pluginId: string, + env?: NodeJS.ProcessEnv, +): { liveEntries: number; maxEntries: number } { + return { + liveEntries: countPluginStateLiveEntries(pluginId, env), + maxEntries: resolveMaxPluginStateEntriesPerPlugin(), + }; +} + function seedPluginStateDatabaseEntriesForTests( entries: readonly PluginStateSeedEntryForTests[], ): void { diff --git a/src/plugin-state/plugin-state-store.test.ts b/src/plugin-state/plugin-state-store.test.ts index 165456f8fcd..e2c43591bd2 100644 --- a/src/plugin-state/plugin-state-store.test.ts +++ b/src/plugin-state/plugin-state-store.test.ts @@ -15,6 +15,8 @@ import { createCorePluginStateSyncKeyedStore, createPluginStateKeyedStore, createPluginStateSyncKeyedStore, + pluginStateEntriesInKeyRange, + registerPluginStateSyncSequencedJournalEntry, resetPluginStateStoreForTests, sweepExpiredPluginStateEntries, } from "./plugin-state-store.js"; @@ -103,6 +105,29 @@ describe("plugin state keyed store", () => { }); }); + it("reads a bounded sortable key range without scanning sibling keys", async () => { + await withPluginStateTestState(async () => { + const store = createPluginStateSyncKeyedStore<{ count: number }>("memory-core", { + namespace: "events", + maxEntries: 10, + }); + store.register("workspace:event:0001", { count: 1 }); + store.register("workspace:event:0002", { count: 2 }); + store.register("workspace:other:0003", { count: 3 }); + + expect( + pluginStateEntriesInKeyRange({ + pluginId: "memory-core", + namespace: "events", + keyStartInclusive: "workspace:event:", + keyEndExclusive: "workspace:event;", + limit: 1, + order: "desc", + }), + ).toMatchObject([{ key: "workspace:event:0002", value: { count: 2 } }]); + }); + }); + it("updates a key from the current stored value", async () => { await withPluginStateTestState(async () => { const store = createPluginStateSyncKeyedStore<{ count: number }>("discord", { @@ -537,6 +562,74 @@ describe("plugin state keyed store", () => { }); }); + it("sheds sequenced journal rows without evicting durable sibling state", async () => { + await withPluginStateTestState(async () => { + const maxPluginEntries = 40; + setMaxPluginStateEntriesPerPluginForTests(maxPluginEntries); + seedPluginStateEntriesForTests([ + ...Array.from({ length: maxPluginEntries - 3 }, (_, entryIndex) => ({ + pluginId: "memory-core", + namespace: "durable-state", + key: `durable-${entryIndex}`, + value: { entryIndex }, + })), + { + pluginId: "memory-core", + namespace: "memory-host.event-migration-checkpoints", + key: "generation", + value: { kind: "raw-checkpoint" }, + }, + { + pluginId: "memory-core", + namespace: "memory-host.event-cursors", + key: "workspace", + value: { kind: "cursor", lastSequence: 1 }, + }, + { + pluginId: "memory-core", + namespace: "memory-host.events", + key: "event-1", + value: { sequence: 1 }, + }, + ]); + + expect( + registerPluginStateSyncSequencedJournalEntry({ + pluginId: "memory-core", + cursorOptions: { + namespace: "memory-host.event-cursors", + maxEntries: 1_000, + }, + cursorKey: "workspace", + journalOptions: { namespace: "memory-host.events", maxEntries: 10_000 }, + initialSequence: 0, + journalKey: (sequence) => `event-${sequence}`, + journalValue: (sequence) => ({ sequence }), + }), + ).toBe(2); + + const durable = createPluginStateKeyedStore("memory-core", { + namespace: "durable-state", + maxEntries: maxPluginEntries, + }); + const checkpoints = createPluginStateKeyedStore("memory-core", { + namespace: "memory-host.event-migration-checkpoints", + maxEntries: 10_000, + overflowPolicy: "reject-new", + }); + const journal = createPluginStateKeyedStore("memory-core", { + namespace: "memory-host.events", + maxEntries: 10_000, + }); + await expect(durable.lookup("durable-0")).resolves.toEqual({ entryIndex: 0 }); + await expect(checkpoints.lookup("generation")).resolves.toEqual({ + kind: "raw-checkpoint", + }); + await expect(journal.lookup("event-1")).resolves.toBeUndefined(); + await expect(journal.lookup("event-2")).resolves.toEqual({ sequence: 2 }); + }); + }); + it("leaves room for Telegram sibling namespaces at their persistent budgets", async () => { await withPluginStateTestState(async () => { seedPluginStateEntriesForTests([ diff --git a/src/plugin-state/plugin-state-store.ts b/src/plugin-state/plugin-state-store.ts index e9f7b0b434f..3cf3ed93fef 100644 --- a/src/plugin-state/plugin-state-store.ts +++ b/src/plugin-state/plugin-state-store.ts @@ -12,6 +12,7 @@ import { pluginStateLookup, pluginStateRegister, pluginStateRegisterIfAbsent, + pluginStateRegisterSequencedJournalEntry, pluginStateUpdate, } from "./plugin-state-store.sqlite.js"; import type { @@ -44,8 +45,10 @@ export type { export { closePluginStateDatabase, countPluginStateLiveEntries, + getPluginStateCapacity, isPluginStateDatabaseOpen, MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN, + pluginStateEntriesInKeyRange, resolveMaxPluginStateEntriesPerPlugin, sweepExpiredPluginStateEntries, } from "./plugin-state-store.sqlite.js"; @@ -62,6 +65,12 @@ type PreparedRegisterParams = { ttlMs?: number; }; +type PluginStateImportEntry = { + key: string; + value: unknown; + createdAt: number; +}; + const namespaceOptionSignatures = new Map(); function invalidInput( message: string, @@ -466,6 +475,121 @@ export function createPluginStateSyncKeyedStore( return createSyncKeyedStoreForPluginId(pluginId, options); } +/** Atomically allocates a workspace sequence and appends one journal entry. */ +export function registerPluginStateSyncSequencedJournalEntry(params: { + pluginId: string; + cursorOptions: OpenKeyedStoreOptions; + cursorKey: string; + journalOptions: OpenKeyedStoreOptions; + initialSequence: number; + journalKey: (sequence: number) => string; + journalValue: (sequence: number) => unknown; +}): number { + if (params.pluginId.startsWith("core:")) { + throw invalidInput("Plugin ids starting with 'core:' are reserved for core consumers.", "open"); + } + if (!Number.isSafeInteger(params.initialSequence) || params.initialSequence < 0) { + throw invalidInput("plugin state initial journal sequence must be a safe non-negative integer"); + } + const cursorNamespace = validateNamespace(params.cursorOptions.namespace); + const cursorMaxEntries = validateMaxEntries(params.cursorOptions.maxEntries); + const cursorOverflowPolicy = validateOverflowPolicy(params.cursorOptions.overflowPolicy); + const cursorDefaultTtlMs = validateOptionalTtlMs(params.cursorOptions.defaultTtlMs); + const journalNamespace = validateNamespace(params.journalOptions.namespace); + const journalMaxEntries = validateMaxEntries(params.journalOptions.maxEntries); + const journalOverflowPolicy = validateOverflowPolicy(params.journalOptions.overflowPolicy); + const journalDefaultTtlMs = validateOptionalTtlMs(params.journalOptions.defaultTtlMs); + if ( + cursorOverflowPolicy !== "evict-oldest" || + journalOverflowPolicy !== "evict-oldest" || + cursorDefaultTtlMs !== undefined || + journalDefaultTtlMs !== undefined + ) { + throw invalidInput("sequenced plugin state journals require non-expiring evict-oldest stores"); + } + if (params.cursorOptions.env !== params.journalOptions.env) { + throw invalidInput("sequenced plugin state journal stores must share one environment"); + } + const cursorKey = validateKey(params.cursorKey); + assertConsistentOptions(params.pluginId, cursorNamespace, { + maxEntries: cursorMaxEntries, + overflowPolicy: cursorOverflowPolicy, + defaultTtlMs: cursorDefaultTtlMs, + }); + assertConsistentOptions(params.pluginId, journalNamespace, { + maxEntries: journalMaxEntries, + overflowPolicy: journalOverflowPolicy, + defaultTtlMs: journalDefaultTtlMs, + }); + return pluginStateRegisterSequencedJournalEntry({ + pluginId: params.pluginId, + cursorNamespace, + cursorKey, + cursorMaxEntries, + journalNamespace, + journalMaxEntries, + initialSequence: params.initialSequence, + readCursorSequence(valueJson) { + try { + const value = JSON.parse(valueJson) as { kind?: unknown; lastSequence?: unknown }; + return value.kind === "cursor" && Number.isSafeInteger(value.lastSequence) + ? (value.lastSequence as number) + : undefined; + } catch { + return undefined; + } + }, + prepareEntry(sequence) { + const cursor = prepareRegisterParams(cursorKey, { kind: "cursor", lastSequence: sequence }); + const journal = prepareRegisterParams( + params.journalKey(sequence), + params.journalValue(sequence), + ); + return { + cursorValueJson: cursor.valueJson, + journalKey: journal.key, + journalValueJson: journal.valueJson, + }; + }, + ...(params.cursorOptions.env ? { env: params.cursorOptions.env } : {}), + }); +} + +/** Doctor-only import that preserves source age for retention ordering. */ +export function importPluginStateEntriesForDoctor( + pluginId: string, + options: OpenKeyedStoreOptions, + entries: readonly PluginStateImportEntry[], +): void { + if (pluginId.startsWith("core:")) { + throw invalidInput("Plugin ids starting with 'core:' are reserved for core consumers.", "open"); + } + const namespace = validateNamespace(options.namespace); + const maxEntries = validateMaxEntries(options.maxEntries); + const overflowPolicy = validateOverflowPolicy(options.overflowPolicy); + const defaultTtlMs = validateOptionalTtlMs(options.defaultTtlMs); + const env = options.env; + assertConsistentOptions(pluginId, namespace, { maxEntries, overflowPolicy, defaultTtlMs }); + + for (const entry of entries) { + if (!Number.isSafeInteger(entry.createdAt)) { + throw invalidInput("plugin state import createdAt must be a safe integer", "register"); + } + const prepared = prepareRegisterParams(entry.key, entry.value, defaultTtlMs); + pluginStateRegister({ + pluginId, + namespace, + key: prepared.key, + valueJson: prepared.valueJson, + maxEntries, + overflowPolicy, + createdAtMs: entry.createdAt, + ...(env ? { env } : {}), + ...(prepared.ttlMs != null ? { ttlMs: prepared.ttlMs } : {}), + }); + } +} + /** Opens a sync plugin-state namespace for a trusted core owner id. */ export function createCorePluginStateSyncKeyedStore( options: OpenKeyedStoreOptions & { ownerId: `core:${string}` }, diff --git a/src/plugins/doctor-contract-registry.ts b/src/plugins/doctor-contract-registry.ts index 44c23e12398..13cf0e2686a 100644 --- a/src/plugins/doctor-contract-registry.ts +++ b/src/plugins/doctor-contract-registry.ts @@ -59,11 +59,20 @@ export type PluginDoctorStateMigrationDetection = { export type PluginDoctorStateMigrationContext = { openPluginStateKeyedStore: (options: OpenKeyedStoreOptions) => PluginStateKeyedStore; + /** Doctor-only batch import preserving source age for retention ordering. */ + importPluginStateEntries?: ( + options: OpenKeyedStoreOptions, + entries: readonly { key: string; value: unknown; createdAt: number }[], + ) => void; + /** Plugin-wide live-row capacity for import preflight. Older test hosts may omit it. */ + getPluginStateCapacity?: () => { liveEntries: number; maxEntries: number }; }; export type PluginDoctorStateMigration = { id: string; label: string; + /** Import retired file state only during explicit `doctor --fix` repair. */ + doctorOnly?: boolean; detectLegacyState: (params: { config: OpenClawConfig; env: NodeJS.ProcessEnv; @@ -216,6 +225,7 @@ function coercePluginDoctorStateMigrations(value: unknown): PluginDoctorStateMig return value.filter(isPluginDoctorStateMigration).map((migration) => ({ id: migration.id.trim(), label: migration.label.trim(), + doctorOnly: migration.doctorOnly === true ? true : undefined, detectLegacyState: migration.detectLegacyState, migrateLegacyState: migration.migrateLegacyState, })); diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index 2aa6b51ddcc..d27589adf05 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -10,6 +10,14 @@ export type Generated = ? ColumnType : ColumnType; +export interface AcpParentStreamEvents { + created_at: number; + event_json: string; + run_id: string; + seq: number; + session_id: string; +} + export interface AuthProfileState { state_json: string; state_key: string; @@ -278,6 +286,7 @@ export interface TranscriptEvents { } export interface DB { + acp_parent_stream_events: AcpParentStreamEvents; auth_profile_state: AuthProfileState; auth_profile_store: AuthProfileStore; cache_entries: CacheEntries; diff --git a/src/state/openclaw-agent-db.test.ts b/src/state/openclaw-agent-db.test.ts index 217cd74329b..ea6c81156a9 100644 --- a/src/state/openclaw-agent-db.test.ts +++ b/src/state/openclaw-agent-db.test.ts @@ -513,8 +513,8 @@ describe("openclaw agent database", () => { expect(registered?.sizeBytes).toBeGreaterThan(0); }); - it("folds additive heartbeat storage into schema version 11", () => { - expect(OPENCLAW_AGENT_SCHEMA_VERSION).toBe(11); + it("keeps additive heartbeat repair while upgrading schema version 11", () => { + expect(OPENCLAW_AGENT_SCHEMA_VERSION).toBe(12); const stateDir = createTempStateDir(); const env = { OPENCLAW_STATE_DIR: stateDir }; const opened = openOpenClawAgentDatabase({ agentId: "worker-1", env }); @@ -536,12 +536,12 @@ describe("openclaw agent database", () => { .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") .get("heartbeat_outcomes"), ).toEqual({ name: "heartbeat_outcomes" }); - expect(readSqliteNumberPragma(reopened.db, "user_version")).toBe(11); + expect(readSqliteNumberPragma(reopened.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); expect( reopened.db .prepare("SELECT schema_version FROM schema_meta WHERE meta_key = 'primary'") .get(), - ).toEqual({ schema_version: 11 }); + ).toEqual({ schema_version: OPENCLAW_AGENT_SCHEMA_VERSION }); }); it("upgrades version 10 with agent state intact and adds lease storage", () => { @@ -589,6 +589,47 @@ describe("openclaw agent database", () => { ).toEqual({ name: "state_leases" }); }); + it("upgrades version 11 with agent state intact and adds ACP parent-stream storage", () => { + const stateDir = createTempStateDir(); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const opened = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + const databasePath = opened.path; + opened.db + .prepare( + "INSERT INTO auth_profile_state (state_key, state_json, updated_at) VALUES (?, ?, ?)", + ) + .run("last-good", '{"profile":"primary"}', 10); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + + const { DatabaseSync } = requireNodeSqlite(); + const legacy = new DatabaseSync(databasePath); + legacy.exec(` + DROP INDEX idx_agent_acp_parent_stream_run; + DROP TABLE acp_parent_stream_events; + PRAGMA user_version = 11; + UPDATE schema_meta SET schema_version = 11 WHERE meta_key = 'primary'; + `); + legacy.close(); + + const migrated = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + expect(migrated.db.prepare("PRAGMA user_version").get()).toEqual({ + user_version: OPENCLAW_AGENT_SCHEMA_VERSION, + }); + expect( + migrated.db + .prepare("SELECT state_json FROM auth_profile_state WHERE state_key = ?") + .get("last-good"), + ).toEqual({ state_json: '{"profile":"primary"}' }); + expect( + migrated.db + .prepare( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = 'acp_parent_stream_events'", + ) + .get(), + ).toEqual({ name: "acp_parent_stream_events" }); + }); + it("migrates version 8 tables to STRICT without losing agent state", () => { const stateDir = createTempStateDir(); const env = { OPENCLAW_STATE_DIR: stateDir }; diff --git a/src/state/openclaw-agent-db.ts b/src/state/openclaw-agent-db.ts index f69702dad4c..8f12922b308 100644 --- a/src/state/openclaw-agent-db.ts +++ b/src/state/openclaw-agent-db.ts @@ -83,6 +83,7 @@ export { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js"; * per pathname, protected with private file modes, and registered in the shared * OpenClaw state database for discovery and maintenance. */ +// v12 = session-owned ACP parent-stream events. // v11 = agent-scoped runtime leases, durable delivery operations, canonical // external conversation addresses, and bounded per-session heartbeat outcome context. // v10 = materialized active transcript paths. @@ -93,7 +94,7 @@ export { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js"; // The v4 session/transcript flip and main's v2 memory-identity // change is folded in structure-gated (migrateMemoryIndexSourcesIdentity), so // v2 main DBs and pre-merge v4 flip DBs both converge on this schema. -export const OPENCLAW_AGENT_SCHEMA_VERSION = 11; +export const OPENCLAW_AGENT_SCHEMA_VERSION = 12; const OPENCLAW_AGENT_DB_DIR_MODE = 0o700; const OPENCLAW_AGENT_DB_FILE_MODE = 0o600; const OPENCLAW_AGENT_DB_SLOW_OPEN_MS = 1_000; diff --git a/src/state/openclaw-agent-schema.generated.ts b/src/state/openclaw-agent-schema.generated.ts index 3f4692b4f19..0796f5fa95f 100644 --- a/src/state/openclaw-agent-schema.generated.ts +++ b/src/state/openclaw-agent-schema.generated.ts @@ -222,6 +222,19 @@ CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run ON trajectory_runtime_events(session_id, run_id, seq) WHERE run_id IS NOT NULL; +CREATE TABLE IF NOT EXISTS acp_parent_stream_events ( + session_id TEXT NOT NULL, + run_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, run_id, seq), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_agent_acp_parent_stream_run + ON acp_parent_stream_events(run_id, seq); + CREATE TABLE IF NOT EXISTS transcript_event_identities ( session_id TEXT NOT NULL, event_id TEXT NOT NULL, diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index a46b1697100..1df4c9b2df5 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -217,6 +217,19 @@ CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run ON trajectory_runtime_events(session_id, run_id, seq) WHERE run_id IS NOT NULL; +CREATE TABLE IF NOT EXISTS acp_parent_stream_events ( + session_id TEXT NOT NULL, + run_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, run_id, seq), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_agent_acp_parent_stream_run + ON acp_parent_stream_events(run_id, seq); + CREATE TABLE IF NOT EXISTS transcript_event_identities ( session_id TEXT NOT NULL, event_id TEXT NOT NULL, diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts index 7c5fc0ad667..0f755917944 100644 --- a/src/state/openclaw-state-db.generated.d.ts +++ b/src/state/openclaw-state-db.generated.d.ts @@ -501,6 +501,7 @@ export interface DiagnosticEvents { event_key: string; payload_json: string; scope: string; + sequence: Generated; } export interface DiagnosticStabilityBundles { diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 1d54472345e..9c7ce90d9ab 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -2336,6 +2336,56 @@ describe("openclaw state database", () => { ); }); + it("backfills diagnostic event sequences in legacy creation order", () => { + const stateDir = createTempStateDir(); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + const databasePath = openOpenClawStateDatabase(options).path; + closeOpenClawStateDatabaseForTest(); + + const { DatabaseSync } = requireNodeSqlite(); + const legacyDb = new DatabaseSync(databasePath); + legacyDb.exec(` + DROP INDEX idx_diagnostic_events_scope_sequence; + ALTER TABLE diagnostic_events DROP COLUMN sequence; + CREATE INDEX idx_diagnostic_events_scope_created + ON diagnostic_events(scope, created_at, event_key); + INSERT INTO diagnostic_events (scope, event_key, payload_json, created_at) VALUES + ('alpha', 'late', '{}', 20), + ('alpha', 'tie-first', '{}', 10), + ('alpha', 'tie-second', '{}', 10), + ('beta', 'only', '{}', 30); + `); + legacyDb.close(); + + const reopened = openOpenClawStateDatabase(options); + const rows = reopened.db + .prepare( + `SELECT scope, event_key, sequence + FROM diagnostic_events + ORDER BY scope, sequence`, + ) + .all(); + expect(rows).toEqual([ + { scope: "alpha", event_key: "tie-first", sequence: 1 }, + { scope: "alpha", event_key: "tie-second", sequence: 2 }, + { scope: "alpha", event_key: "late", sequence: 3 }, + { scope: "beta", event_key: "only", sequence: 1 }, + ]); + const indexes = reopened.db.prepare("PRAGMA index_list(diagnostic_events)").all() as Array<{ + name?: unknown; + }>; + expect(indexes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "idx_diagnostic_events_scope_sequence" }), + ]), + ); + expect(indexes).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "idx_diagnostic_events_scope_created" }), + ]), + ); + }); + it("adds relay origins to existing APNs registration tables", () => { const stateDir = createTempStateDir(); const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index bbe873981a8..f41892ac3db 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -94,6 +94,7 @@ const OPENCLAW_STATE_CANONICAL_UNIQUE_INDEXES = [ ] as const satisfies readonly CanonicalSqliteUniqueIndex[]; const OPENCLAW_STATE_MAINTENANCE_SCHEMA_COMPATIBILITY = { allowedColumnDefinitions: { + "diagnostic_events.sequence": ["sequence INTEGER NOT NULL DEFAULT 0"], "commitments.attempts": ["attempts INTEGER NOT NULL DEFAULT 0"], "commitments.confidence": ["confidence REAL NOT NULL DEFAULT 0"], "commitments.created_at_ms": ["created_at_ms INTEGER NOT NULL DEFAULT 0"], @@ -1446,6 +1447,33 @@ function backfillDeliveryQueueEntriesFromEntryJson(db: DatabaseSync): void { // The caller owns the state.schema.ensure transaction so every probe, DDL // change, and backfill observes one authoritative schema across processes. function ensureAdditiveStateColumns(db: DatabaseSync): void { + const addedDiagnosticEventSequence = ensureColumn( + db, + "diagnostic_events", + "sequence INTEGER NOT NULL DEFAULT 0", + ); + if (addedDiagnosticEventSequence) { + // Preserve the legacy (created_at, rowid) order before the new sequence + // index becomes authoritative, including stable ties within each scope. + db.exec(` + WITH ranked AS ( + SELECT + rowid AS event_rowid, + ROW_NUMBER() OVER ( + PARTITION BY scope + ORDER BY created_at ASC, rowid ASC + ) AS sequence + FROM diagnostic_events + ) + UPDATE diagnostic_events + SET sequence = ( + SELECT ranked.sequence + FROM ranked + WHERE ranked.event_rowid = diagnostic_events.rowid + ); + `); + } + db.exec("DROP INDEX IF EXISTS idx_diagnostic_events_scope_created;"); ensureColumn(db, "worktrees", "provisioned_paths_json TEXT"); ensureColumn(db, "node_host_config", "gateway_context_path TEXT"); ensureColumn(db, "node_host_config", "installed_apps_sharing INTEGER NOT NULL DEFAULT 0"); diff --git a/src/state/openclaw-state-lease.ts b/src/state/openclaw-state-lease.ts index 43cfb8cf15a..310bf2bd894 100644 --- a/src/state/openclaw-state-lease.ts +++ b/src/state/openclaw-state-lease.ts @@ -18,13 +18,16 @@ import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.gene import { openOpenClawStateDatabase, runOpenClawStateWriteTransaction, + type OpenClawStateDatabaseOptions, } from "./openclaw-state-db.js"; type LeaseDatabase = Pick; type AgentLeaseDatabase = Pick; type LeaseKysely = ReturnType>; -type OpenClawStateLeaseDatabase = { scope: "shared" } | { scope: "agent"; agentId: string }; +type OpenClawStateLeaseDatabase = + | { scope: "shared"; options?: OpenClawStateDatabaseOptions } + | { scope: "agent"; agentId: string }; type OpenClawStateLeaseOptions = { scope: string; @@ -175,11 +178,11 @@ function withLeaseWriteTransaction( busyTimeoutMs = LEASE_DB_BUSY_TIMEOUT_MS, ): T { if (database.scope === "shared") { - const stateDatabase = openOpenClawStateDatabase(); + const stateDatabase = openOpenClawStateDatabase(database.options); const run = () => runOpenClawStateWriteTransaction( ({ db }) => operation(db, getNodeSqliteKysely(db)), - {}, + database.options, { operationLabel, busyTimeoutMs }, ); return withBusyTimeout(stateDatabase.db, busyTimeoutMs, run); @@ -200,7 +203,7 @@ function withLeaseRead( ): T { const sqlite = database.scope === "shared" - ? openOpenClawStateDatabase().db + ? openOpenClawStateDatabase(database.options).db : openOpenClawAgentDatabase({ agentId: database.agentId }).db; return operation(sqlite, getNodeSqliteKysely(sqlite)); } diff --git a/src/state/openclaw-state-schema.generated.ts b/src/state/openclaw-state-schema.generated.ts index 966ee121629..ddbd5be2e03 100644 --- a/src/state/openclaw-state-schema.generated.ts +++ b/src/state/openclaw-state-schema.generated.ts @@ -27,11 +27,12 @@ CREATE TABLE IF NOT EXISTS diagnostic_events ( event_key TEXT NOT NULL, payload_json TEXT NOT NULL, created_at INTEGER NOT NULL, + sequence INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (scope, event_key) ) STRICT; -CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created - ON diagnostic_events(scope, created_at, event_key); +CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_sequence + ON diagnostic_events(scope, sequence, event_key); CREATE TABLE IF NOT EXISTS skill_usage ( skill_file TEXT NOT NULL PRIMARY KEY, diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql index bf7b2548c7f..e7f1d957890 100644 --- a/src/state/openclaw-state-schema.sql +++ b/src/state/openclaw-state-schema.sql @@ -22,11 +22,12 @@ CREATE TABLE IF NOT EXISTS diagnostic_events ( event_key TEXT NOT NULL, payload_json TEXT NOT NULL, created_at INTEGER NOT NULL, + sequence INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (scope, event_key) ) STRICT; -CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created - ON diagnostic_events(scope, created_at, event_key); +CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_sequence + ON diagnostic_events(scope, sequence, event_key); CREATE TABLE IF NOT EXISTS skill_usage ( skill_file TEXT NOT NULL PRIMARY KEY, diff --git a/src/system-agent/audit.test-support.ts b/src/system-agent/audit.test-support.ts new file mode 100644 index 00000000000..6ea2a4ca58c --- /dev/null +++ b/src/system-agent/audit.test-support.ts @@ -0,0 +1,14 @@ +import { createSqliteAuditRecordStore } from "../infra/sqlite-audit-record-store.js"; +import { + SYSTEM_AGENT_AUDIT_MAX_ENTRIES, + SYSTEM_AGENT_AUDIT_SCOPE, + type SystemAgentAuditEntry, +} from "./audit.js"; + +export function listSystemAgentAuditEntriesForTests(params?: { env?: NodeJS.ProcessEnv }) { + return createSqliteAuditRecordStore({ + scope: SYSTEM_AGENT_AUDIT_SCOPE, + maxEntries: SYSTEM_AGENT_AUDIT_MAX_ENTRIES, + ...(params?.env ? { env: params.env } : {}), + }).entries(); +} diff --git a/src/system-agent/audit.test.ts b/src/system-agent/audit.test.ts index 6a14181393a..3c816133a7f 100644 --- a/src/system-agent/audit.test.ts +++ b/src/system-agent/audit.test.ts @@ -1,13 +1,15 @@ -// OpenClaw audit tests cover filesystem-backed rescue audit scenarios. -import fs from "node:fs/promises"; +// OpenClaw audit tests cover SQLite-backed rescue audit scenarios. import { afterEach, describe, expect, it, vi } from "vitest"; +import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; import { withTempDir } from "../test-helpers/temp-dir.js"; -import { appendSystemAgentAuditEntry, resolveSystemAgentAuditPath } from "./audit.js"; +import { appendSystemAgentAuditEntry, SYSTEM_AGENT_AUDIT_STORE_LABEL } from "./audit.js"; +import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js"; describe("OpenClaw audit log", () => { const previousStateDir = process.env.OPENCLAW_STATE_DIR; afterEach(() => { + resetPluginStateStoreForTests(); if (previousStateDir === undefined) { delete process.env.OPENCLAW_STATE_DIR; } else { @@ -15,21 +17,25 @@ describe("OpenClaw audit log", () => { } }); - it("writes jsonl records under the OpenClaw audit dir", async () => { + it("writes records into shared SQLite state", async () => { await withTempDir({ prefix: "openclaw-audit-" }, async (tempDir) => { vi.stubEnv("OPENCLAW_STATE_DIR", tempDir); - const auditPath = await appendSystemAgentAuditEntry({ + const auditStore = await appendSystemAgentAuditEntry({ operation: "config.setDefaultModel", summary: "Set default model to openai/gpt-5.2", configHashBefore: "before", configHashAfter: "after", }); - expect(auditPath).toBe(resolveSystemAgentAuditPath()); - const lines = (await fs.readFile(auditPath, "utf8")).trim().split("\n"); - expect(lines).toHaveLength(1); - const entry = JSON.parse(lines[0] ?? "{}") as Record; + expect(auditStore).toBe(SYSTEM_AGENT_AUDIT_STORE_LABEL); + const records = listSystemAgentAuditEntriesForTests(); + expect(records).toHaveLength(1); + const entry = records[0]?.value; + expect(entry).toBeDefined(); + if (!entry) { + throw new Error("expected persisted system-agent audit entry"); + } expect(entry.operation).toBe("config.setDefaultModel"); expect(entry.summary).toBe("Set default model to openai/gpt-5.2"); expect(entry.configHashBefore).toBe("before"); diff --git a/src/system-agent/audit.ts b/src/system-agent/audit.ts index effa5d6c474..eb23820d55e 100644 --- a/src/system-agent/audit.ts +++ b/src/system-agent/audit.ts @@ -1,16 +1,15 @@ -// OpenClaw audit helpers append JSONL records for approved local-state changes. -import fs from "node:fs/promises"; -import path from "node:path"; -import { resolveStateDir } from "../config/paths.js"; -import { appendRegularFile } from "../infra/fs-safe.js"; +// OpenClaw audit helpers persist approved local-state changes. +import { randomUUID } from "node:crypto"; +import { createSqliteAuditRecordStore } from "../infra/sqlite-audit-record-store.js"; +import { redactSecrets } from "../logging/redact.js"; /** * Append-only audit log helpers for OpenClaw writes. * * Discovery and read-only commands stay quiet; persistent operations append a - * JSONL entry under the state directory with config hashes and redacted details. + * SQLite entry under the shared state directory with config hashes and redacted details. */ -type SystemAgentAuditEntry = { +export type SystemAgentAuditEntry = { timestamp: string; operation: string; summary: string; @@ -20,30 +19,32 @@ type SystemAgentAuditEntry = { details?: Record; }; -/** Resolve the JSONL audit path for OpenClaw persistent operations. */ -export function resolveSystemAgentAuditPath( - env: NodeJS.ProcessEnv = process.env, - stateDir = resolveStateDir(env), -): string { - return path.join(stateDir, "audit", "system-agent.jsonl"); +export const SYSTEM_AGENT_AUDIT_SCOPE = "system-agent-audit"; +export const SYSTEM_AGENT_AUDIT_MAX_ENTRIES = 50_000; +export const SYSTEM_AGENT_AUDIT_STORE_LABEL = + "SQLite diagnostic_events/system-agent-audit state (latest 50000 rows)"; + +function openSystemAgentAuditStore(env?: NodeJS.ProcessEnv) { + return createSqliteAuditRecordStore({ + scope: SYSTEM_AGENT_AUDIT_SCOPE, + maxEntries: SYSTEM_AGENT_AUDIT_MAX_ENTRIES, + ...(env ? { env } : {}), + }); } -/** Append one OpenClaw audit entry and return the file path written. */ +/** Append one OpenClaw audit entry and return its SQLite owner label. */ export async function appendSystemAgentAuditEntry( entry: Omit, - opts: { env?: NodeJS.ProcessEnv; auditPath?: string } = {}, + opts: { env?: NodeJS.ProcessEnv } = {}, ): Promise { - const auditPath = opts.auditPath ?? resolveSystemAgentAuditPath(opts.env); - await fs.mkdir(path.dirname(auditPath), { recursive: true }); - const line = JSON.stringify({ + const record = redactSecrets({ timestamp: new Date().toISOString(), ...entry, } satisfies SystemAgentAuditEntry); - // Audit writes reject symlinked parents so approval records cannot be redirected silently. - await appendRegularFile({ - filePath: auditPath, - content: `${line}\n`, - rejectSymlinkParents: true, - }); - return auditPath; + openSystemAgentAuditStore(opts.env).register( + `${record.timestamp}:${randomUUID()}`, + record, + Date.parse(record.timestamp), + ); + return SYSTEM_AGENT_AUDIT_STORE_LABEL; } diff --git a/src/system-agent/operations-execute.ts b/src/system-agent/operations-execute.ts index 33f147c41b0..37b979c490d 100644 --- a/src/system-agent/operations-execute.ts +++ b/src/system-agent/operations-execute.ts @@ -5,7 +5,7 @@ import type { RuntimeEnv } from "../runtime.js"; import { resolveUserPath, shortenHomePath } from "../utils.js"; import { t } from "../wizard/i18n/index.js"; import { isReservedSystemAgentId } from "./agent-id.js"; -import { resolveSystemAgentAuditPath } from "./audit.js"; +import { SYSTEM_AGENT_AUDIT_STORE_LABEL } from "./audit.js"; import { CONFIG_GET_OUTPUT_MAX_CHARS, CONFIG_SCHEMA_CHILDREN_MAX, @@ -108,7 +108,7 @@ export async function executeSystemAgentOperation( return { applied: false }; } case "audit": - runtime.log(`Audit log: ${resolveSystemAgentAuditPath()}`); + runtime.log(`Audit state: ${SYSTEM_AGENT_AUDIT_STORE_LABEL}`); runtime.log("Only applied writes/actions are recorded; discovery stays quiet."); return { applied: false }; case "config-validate": { diff --git a/src/system-agent/operations.setup.test.ts b/src/system-agent/operations.setup.test.ts index 6eb5fd6a9aa..55b6de9ea06 100644 --- a/src/system-agent/operations.setup.test.ts +++ b/src/system-agent/operations.setup.test.ts @@ -3,19 +3,17 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; +import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js"; import { SystemAgentInferenceUnavailableError } from "./inference-error.js"; import { executeSystemAgentOperation, isPersistentSystemAgentOperation } from "./operations.js"; import { createSystemAgentTestRuntime } from "./system-agent.test-helpers.js"; type TestConfig = Record; -function parseLastJsonLine(raw: string): unknown { - const lastLine = raw.trim().split("\n").at(-1); - if (!lastLine) { - throw new Error("Expected audit log to contain at least one JSON line"); - } - return JSON.parse(lastLine) as unknown; +function readLastAuditEntry(): unknown { + return listSystemAgentAuditEntriesForTests().at(-1)?.value; } function requireRecord(value: unknown, label: string): Record { @@ -172,6 +170,7 @@ describe("parseSystemAgentOperation", () => { }); afterEach(() => { + resetPluginStateStoreForTests(); stateDirSnapshot?.restore(); vi.unstubAllEnvs(); }); @@ -232,8 +231,7 @@ describe("parseSystemAgentOperation", () => { { commit: expect.any(Function) }, ); expect(lines.join("\n")).toContain("Default model: openai/gpt-5.5 (verified and kept)"); - const auditPath = path.join(tempDir, "audit", "system-agent.jsonl"); - const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()); + const audit = readLastAuditEntry(); expectAuditRecord( audit, { @@ -556,9 +554,7 @@ describe("parseSystemAgentOperation", () => { expect(persisted.channels).toEqual({ telegram: { enabled: true } }); expect(lines.join("\n")).toContain("Default model: openai/gpt-5.5"); - const audit = parseLastJsonLine( - await fs.readFile(path.join(tempDir, "audit", "system-agent.jsonl"), "utf8"), - ); + const audit = readLastAuditEntry(); expectAuditRecord( audit, { diff --git a/src/system-agent/operations.test.ts b/src/system-agent/operations.test.ts index 2f0a508b996..ba5fce87bd0 100644 --- a/src/system-agent/operations.test.ts +++ b/src/system-agent/operations.test.ts @@ -3,8 +3,10 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; import type { RuntimeEnv } from "../runtime.js"; import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; +import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js"; import { describeSystemAgentPersistentOperation, executeSystemAgentOperation, @@ -38,6 +40,10 @@ function expectAuditRecord( expectRecordFields(requireRecord(auditRecord.details, "audit details"), detailFields); } +function readLastAuditEntry(): unknown { + return listSystemAgentAuditEntriesForTests().at(-1)?.value; +} + function requireFirstMockCall(mock: unknown, label: string): unknown[] { const call = (mock as { mock?: { calls?: unknown[][] } }).mock?.calls?.[0]; if (!call) { @@ -181,6 +187,7 @@ describe("parseSystemAgentOperation", () => { }); afterEach(() => { + resetPluginStateStoreForTests(); stateDirSnapshot?.restore(); vi.unstubAllEnvs(); }); @@ -624,8 +631,7 @@ describe("parseSystemAgentOperation", () => { cliOptions: {}, }); expect(lines.join("\n")).toContain("[openclaw] done: config.set"); - const auditPath = path.join(tempDir, "audit", "system-agent.jsonl"); - const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()); + const audit = readLastAuditEntry(); expectAuditRecord( audit, { operation: "config.set", summary: "Set config gateway.port" }, @@ -637,7 +643,7 @@ describe("parseSystemAgentOperation", () => { ); }); - it("reports an audit failure without claiming the committed operation failed", async () => { + it("records SQLite audit state despite a retired audit-directory symlink", async () => { const tempDir = opTempDirs.make("openclaw-audit-warning-"); setTestEnvValue("OPENCLAW_STATE_DIR", tempDir); const redirectedAuditDir = path.join(tempDir, "redirected-audit"); @@ -654,9 +660,7 @@ describe("parseSystemAgentOperation", () => { expect(result.applied).toBe(true); expect(runConfigSet).toHaveBeenCalledOnce(); - expect(lines.join("\n")).toContain( - "Set config gateway.port, but OpenClaw could not record its audit entry:", - ); + expect(readLastAuditEntry()).toMatchObject({ operation: "config.set" }); expect(lines.join("\n")).toContain("[openclaw] done: config.set"); }); @@ -691,8 +695,7 @@ describe("parseSystemAgentOperation", () => { }, }); expect(lines.join("\n")).toContain("[openclaw] done: config.setRef"); - const auditPath = path.join(tempDir, "audit", "system-agent.jsonl"); - const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()); + const audit = readLastAuditEntry(); expectAuditRecord( audit, { @@ -938,8 +941,7 @@ describe("parseSystemAgentOperation", () => { expect(installCall[0]).toBe("clawhub:openclaw-demo"); expectRuntimeArg(installCall[1]); expect(lines.join("\n")).toContain("[openclaw] done: plugin.install"); - const auditPath = path.join(tempDir, "audit", "system-agent.jsonl"); - const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()); + const audit = readLastAuditEntry(); expectAuditRecord( audit, { diff --git a/src/system-agent/rescue-channel.live.test.ts b/src/system-agent/rescue-channel.live.test.ts index ab43fd616f2..3a022e09188 100644 --- a/src/system-agent/rescue-channel.live.test.ts +++ b/src/system-agent/rescue-channel.live.test.ts @@ -5,8 +5,10 @@ import { afterEach, describe, expect, it } from "vitest"; import type { CommandContext } from "../auto-reply/reply/commands-types.js"; import { clearConfigCache } from "../config/config.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js"; import { withTempDir } from "../test-helpers/temp-dir.js"; import { deleteTestEnvValue, setTestEnvValue } from "../test-utils/env.js"; +import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js"; import { runSystemAgentRescueMessage } from "./rescue-message.js"; const originalStateDir = process.env.OPENCLAW_STATE_DIR; @@ -53,6 +55,7 @@ async function runRescue(params: { describeLive("OpenClaw live rescue channel smoke", () => { afterEach(() => { + resetPluginStateStoreForTests(); clearConfigCache(); if (originalStateDir === undefined) { deleteTestEnvValue("OPENCLAW_STATE_DIR"); @@ -105,11 +108,11 @@ describeLive("OpenClaw live rescue channel smoke", () => { throw new Error("expected default model object"); } expect(defaultModel.primary).toBe("openai/gpt-5.5"); - const auditPath = path.join(tempDir, "audit", "system-agent.jsonl"); - const auditLines = (await fs.readFile(auditPath, "utf8")).trim().split("\n"); - expect(auditLines.some((line) => line.includes('"operation":"config.setDefaultModel"'))).toBe( - true, - ); + expect( + listSystemAgentAuditEntriesForTests().some( + (entry) => entry.value.operation === "config.setDefaultModel", + ), + ).toBe(true); }); }); }); diff --git a/src/system-agent/rescue-message.test.ts b/src/system-agent/rescue-message.test.ts index 29b225dcf77..acb09d8ceba 100644 --- a/src/system-agent/rescue-message.test.ts +++ b/src/system-agent/rescue-message.test.ts @@ -11,6 +11,7 @@ import { } from "../plugin-state/plugin-state-store.js"; import type { RuntimeEnv } from "../runtime.js"; import { withEnvAsync } from "../test-utils/env.js"; +import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js"; import { extractSystemAgentRescueMessage, runSystemAgentRescueMessage } from "./rescue-message.js"; let tempRoot = ""; @@ -18,6 +19,10 @@ let tempDirId = 0; type TestConfig = Record; +function readLastAuditEntry(): Record { + return (listSystemAgentAuditEntriesForTests().at(-1)?.value ?? {}) as Record; +} + const mockConfig = vi.hoisted(() => { const state = { path: "/tmp/openclaw.json", @@ -531,7 +536,7 @@ describe("OpenClaw rescue message", () => { }); it("queues and applies persistent writes through conversational approval", async () => { - await withRescueStateDir("models-", async (tempDir) => { + await withRescueStateDir("models-", async () => { const cfg: OpenClawConfig = { systemAgent: { rescue: { enabled: true } } }; const deps = { verifyInferenceConfig: vi.fn(async () => ({ @@ -552,8 +557,7 @@ describe("OpenClaw rescue message", () => { }; const model = currentConfig.agents?.defaults?.model; expect(typeof model === "string" ? model : model?.primary).toBe("openai/gpt-5.2"); - const auditPath = path.join(tempDir, "audit", "system-agent.jsonl"); - const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()) as { + const audit = readLastAuditEntry() as { details?: { rescue?: boolean; channel?: string; accountId?: string; senderId?: string }; }; expect(audit.details?.rescue).toBe(true); @@ -564,7 +568,7 @@ describe("OpenClaw rescue message", () => { }); it("queues and applies gateway restart through conversational approval", async () => { - await withRescueStateDir("gateway-", async (tempDir) => { + await withRescueStateDir("gateway-", async () => { const cfg: OpenClawConfig = { systemAgent: { rescue: { enabled: true } } }; const deps = { runGatewayRestart: vi.fn(async () => {}) }; @@ -576,8 +580,7 @@ describe("OpenClaw rescue message", () => { ); expect(deps.runGatewayRestart).toHaveBeenCalledTimes(1); - const auditPath = path.join(tempDir, "audit", "system-agent.jsonl"); - const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()) as { + const audit = readLastAuditEntry() as { operation?: string; details?: { rescue?: boolean; channel?: string; senderId?: string }; }; @@ -658,7 +661,7 @@ describe("OpenClaw rescue message", () => { }); it("queues and applies agent creation through conversational approval", async () => { - await withRescueStateDir("agent-", async (tempDir) => { + await withRescueStateDir("agent-", async () => { const cfg: OpenClawConfig = { systemAgent: { rescue: { enabled: true } } }; const deps = { runAgentsAdd: vi.fn(async () => {}) }; @@ -687,8 +690,7 @@ describe("OpenClaw rescue message", () => { }); expect(agentRuntime).toBeTypeOf("object"); expect(agentOptions).toEqual({ hasFlags: true }); - const auditPath = path.join(tempDir, "audit", "system-agent.jsonl"); - const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()) as { + const audit = readLastAuditEntry() as { operation?: string; details?: { rescue?: boolean; diff --git a/test/e2e/qa-lab/runtime/system-agent-first-run-docker-client.ts b/test/e2e/qa-lab/runtime/system-agent-first-run-docker-client.ts index 6f81120bf1d..c7af1d0c475 100644 --- a/test/e2e/qa-lab/runtime/system-agent-first-run-docker-client.ts +++ b/test/e2e/qa-lab/runtime/system-agent-first-run-docker-client.ts @@ -7,7 +7,13 @@ import path from "node:path"; import { shouldStartOnboardingForFreshInstall } from "../../../../dist/cli/run-main.js"; import { clearConfigCache } from "../../../../dist/config/config.js"; import type { OpenClawConfig } from "../../../../dist/config/types.openclaw.js"; +import { createSqliteAuditRecordStore } from "../../../../dist/infra/sqlite-audit-record-store.js"; import type { RuntimeEnv } from "../../../../dist/runtime.js"; +import { + SYSTEM_AGENT_AUDIT_MAX_ENTRIES, + SYSTEM_AGENT_AUDIT_SCOPE, + type SystemAgentAuditEntry, +} from "../../../../dist/system-agent/audit.js"; import { activateSetupInference, verifySetupInference, @@ -367,10 +373,15 @@ async function main() { "OpenClaw persisted an unrelated ambient credential", ); - const auditPath = path.join(stateDir, "audit", "system-agent.jsonl"); - const audit = (await fs.readFile(auditPath, "utf8")).trim(); + const audit = createSqliteAuditRecordStore({ + scope: SYSTEM_AGENT_AUDIT_SCOPE, + maxEntries: SYSTEM_AGENT_AUDIT_MAX_ENTRIES, + }).entries(); for (const operation of spec.auditOperations) { - assert(audit.includes(`"operation":"${operation}"`), `${operation} audit entry missing`); + assert( + audit.some((entry) => entry.value.operation === operation), + `${operation} audit entry missing`, + ); } console.log("OpenClaw first-run Docker E2E passed"); diff --git a/test/scripts/check-database-first-legacy-stores.test.ts b/test/scripts/check-database-first-legacy-stores.test.ts index ec9e4eaee2e..546c9c3db29 100644 --- a/test/scripts/check-database-first-legacy-stores.test.ts +++ b/test/scripts/check-database-first-legacy-stores.test.ts @@ -315,6 +315,23 @@ describe("check-database-first-legacy-stores", () => { expect(violations).toEqual([{ kind: "legacy store filesystem write", line: 4 }]); }); + it("flags runtime writes to retired core audit JSONL stores", () => { + const violations = collectDatabaseFirstLegacyStoreViolations( + ` + import { promises as fs } from "node:fs"; + import path from "node:path"; + await fs.appendFile(path.join(stateDir, "logs", "config-audit.jsonl"), "{}\\n"); + await fs.appendFile(path.join(stateDir, "audit", "system-agent.jsonl"), "{}\\n"); + `, + "src/infra/audit-writer.ts", + ); + + expect(violations).toEqual([ + { kind: "legacy store filesystem write", line: 4 }, + { kind: "legacy store filesystem write", line: 5 }, + ]); + }); + it("flags runtime writes to retired managed-image record JSON", () => { const violations = collectDatabaseFirstLegacyStoreViolations( ` diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 4db953b568c..d9078baa2b1 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -2868,7 +2868,7 @@ describe("scripts/test-projects changed-target routing", () => { includePatterns: expect.arrayContaining(["src/plugin-sdk/access-groups.test.ts"]), }), expect.objectContaining({ - config: "test/vitest/vitest.unit-fast-isolated.config.ts", + config: "test/vitest/vitest.unit-fast-fake-timers.config.ts", includePatterns: ["src/plugin-sdk/memory-host-events.test.ts"], }), expect.objectContaining({ diff --git a/tsdown.config.ts b/tsdown.config.ts index 453b2e308db..7fa3ca1c8c1 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -334,6 +334,8 @@ function buildDockerE2eHarnessEntries(): Record { "commitments/runtime.test-support": "src/commitments/runtime.test-support.ts", "commitments/store": "src/commitments/store.ts", "config/config": "src/config/config.ts", + "infra/sqlite-audit-record-store": "src/infra/sqlite-audit-record-store.ts", + "system-agent/audit": "src/system-agent/audit.ts", "system-agent/system-agent": "src/system-agent/system-agent.ts", "system-agent/rescue-message": "src/system-agent/rescue-message.ts", "system-agent/setup-inference": "src/system-agent/setup-inference.ts", From 4a96c9da31080bc6c49fbd5306efee6ea112c866 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 11:46:40 +0100 Subject: [PATCH 36/76] feat(ui): link Get the apps from the mobile pairing dialog (#110622) --- ui/src/app/app-host.ts | 4 ++++ ui/src/i18n/locales/en.ts | 2 ++ ui/src/pages/nodes/view-pairing.ts | 5 +++++ ui/src/styles/components.css | 26 ++++++++++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/ui/src/app/app-host.ts b/ui/src/app/app-host.ts index 7edec7e11f1..82eafaacd2c 100644 --- a/ui/src/app/app-host.ts +++ b/ui/src/app/app-host.ts @@ -1509,6 +1509,10 @@ class OpenClawShell extends OpenClawLightDomElement { context.overlays.closeDevicePairSetup(); this.navigate("nodes"); }, + onGetApps: () => { + context.overlays.closeDevicePairSetup(); + this.navigate("apps"); + }, })} ${onboarding && activeRoute !== "custodian" ? html` void; onCopy: (setupCode: string) => void; onManageDevices: () => void; + onGetApps: () => void; }; export function renderDevicePairSetup(props: DevicePairSetupProps) { @@ -40,6 +41,10 @@ export function renderDevicePairSetup(props: DevicePairSetupProps) {

${title}

${description}

+

+ ${t("nodes.pairing.noApp")} + +

` + : nothing} ${session.loadingChildren && session.children.length === 0 ? html`${t("common.loading")}` : nothing} diff --git a/ui/src/components/app-sidebar-session-navigation.ts b/ui/src/components/app-sidebar-session-navigation.ts index 0374d02d2f4..abdae9f2dbb 100644 --- a/ui/src/components/app-sidebar-session-navigation.ts +++ b/ui/src/components/app-sidebar-session-navigation.ts @@ -53,6 +53,7 @@ export abstract class AppSidebarSessionNavigationElement extends AppSidebarSessi @state() protected selectedSessionKeys: ReadonlySet = new Set(); @state() protected expandedChildSessionKeys: ReadonlySet = new Set(); @state() protected collapsedActiveChildSessionKeys: ReadonlySet = new Set(); + @state() protected fullyShownChildSessionKeys: ReadonlySet = new Set(); @state() protected sessionSortMode: SidebarSessionSortMode = "created"; @state() protected sessionsGrouping: SidebarSessionsGrouping = loadStoredSidebarSessionsGrouping(); @@ -162,7 +163,9 @@ export abstract class AppSidebarSessionNavigationElement extends AppSidebarSessi } return { key: row.key, - label: resolveSessionDisplayName(row.key, row), + label: resolveSessionDisplayName(row.key, row, { + includeSubagentPrefix: !isChild, + }), meta: formatSidebarTimestamp(row.updatedAt), subtitle: resolveSessionWorkSubtitle(row), href: `${pathForRoute("chat", context?.basePath ?? "")}${searchForSession(row.key)}`, @@ -618,8 +621,10 @@ export abstract class AppSidebarSessionNavigationElement extends AppSidebarSessi protected toggleSessionChildren(session: SidebarRecentSession) { const next = new Set(this.expandedChildSessionKeys); const collapsedActive = new Set(this.collapsedActiveChildSessionKeys); + const fullyShown = new Set(this.fullyShownChildSessionKeys); if (this.isSessionChildrenExpanded(session)) { next.delete(session.key); + fullyShown.delete(session.key); if (session.containsActiveDescendant) { collapsedActive.add(session.key); } @@ -643,6 +648,11 @@ export abstract class AppSidebarSessionNavigationElement extends AppSidebarSessi } this.expandedChildSessionKeys = next; this.collapsedActiveChildSessionKeys = collapsedActive; + this.fullyShownChildSessionKeys = fullyShown; + } + + protected showAllSessionChildren(sessionKey: string) { + this.fullyShownChildSessionKeys = new Set(this.fullyShownChildSessionKeys).add(sessionKey); } private projectSessionTree( diff --git a/ui/src/components/app-sidebar.test.ts b/ui/src/components/app-sidebar.test.ts index 29291737c6b..8583d3d859e 100644 --- a/ui/src/components/app-sidebar.test.ts +++ b/ui/src/components/app-sidebar.test.ts @@ -8,6 +8,7 @@ import "../test-helpers/app-sidebar-cases/catalog-live-events.ts"; import "../test-helpers/app-sidebar-cases/catalog-live.ts"; import "../test-helpers/app-sidebar-cases/catalog-live-state.ts"; import "../test-helpers/app-sidebar-cases/catalog-pages.ts"; +import "../test-helpers/app-sidebar-cases/child-sessions-cap.ts"; import "../test-helpers/app-sidebar-cases/child-sessions.ts"; import "../test-helpers/app-sidebar-cases/group-mutations.ts"; import "../test-helpers/app-sidebar-cases/interactions.ts"; diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index 0c5f2ce47f1..6cc9715ca73 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -627,6 +627,7 @@ export const en: TranslationMap = { showChildSessions: "Show {count} child sessions for {session}", hideChildSessions: "Hide {count} child sessions for {session}", childSessions: "Child sessions", + showMoreChildren: "Show {count} more", archived: "Archived", pinned: "Pinned", unread: "Unread", diff --git a/ui/src/lib/session-display.test.ts b/ui/src/lib/session-display.test.ts index 34152566f22..ab6efbda714 100644 --- a/ui/src/lib/session-display.test.ts +++ b/ui/src/lib/session-display.test.ts @@ -77,6 +77,34 @@ describe("resolveSessionDisplayName", () => { "node-fleet-…8b2e", ); }); + + it("can omit only the subagent prefix while preserving its untitled fallback", () => { + const key = "agent:main:subagent:worker"; + expect(resolveSessionDisplayName(key, { label: "Research sources" })).toBe( + "Subagent: Research sources", + ); + expect( + resolveSessionDisplayName( + key, + { label: "Subagent: Research sources" }, + { + includeSubagentPrefix: false, + }, + ), + ).toBe("Research sources"); + expect(resolveSessionDisplayName(key, undefined, { includeSubagentPrefix: false })).toBe( + "Subagent:", + ); + expect( + resolveSessionDisplayName( + "agent:main:cron:daily", + { label: "Daily" }, + { + includeSubagentPrefix: false, + }, + ), + ).toBe("Cron: Daily"); + }); }); describe("resolveSessionWorkSubtitle", () => { diff --git a/ui/src/lib/session-display.ts b/ui/src/lib/session-display.ts index 4826b2d1b1d..c58a4c91f87 100644 --- a/ui/src/lib/session-display.ts +++ b/ui/src/lib/session-display.ts @@ -92,6 +92,10 @@ type SessionDisplayRow = { derivedTitle?: string; } & SessionWorktreeDisplayRow; +type SessionDisplayOptions = { + includeSubagentPrefix?: boolean; +}; + function capitalize(s: string): string { return s.charAt(0).toUpperCase() + s.slice(1); } @@ -169,7 +173,11 @@ function parseSessionKey(key: string): SessionKeyInfo { return { prefix: "", fallbackName: key }; } -export function resolveSessionDisplayName(key: string, row?: SessionDisplayRow): string { +export function resolveSessionDisplayName( + key: string, + row?: SessionDisplayRow, + options: SessionDisplayOptions = {}, +): string { const label = normalizeOptionalString(row?.label) ?? ""; const displayName = normalizeOptionalString(row?.displayName) ?? ""; const derivedTitle = normalizeOptionalString(row?.derivedTitle) ?? ""; @@ -180,6 +188,9 @@ export function resolveSessionDisplayName(key: string, row?: SessionDisplayRow): return name; } const prefixPattern = new RegExp(`^${prefix.replace(/[.*+?^${}()|[\\]\\]/g, "\\$&")}\\s*`, "i"); + if (prefix === "Subagent:" && options.includeSubagentPrefix === false) { + return name.replace(prefixPattern, "").trim() || fallbackName; + } return prefixPattern.test(name) ? name : `${prefix} ${name}`; }; diff --git a/ui/src/styles/layout.css b/ui/src/styles/layout.css index b976dc66f60..996e6cf42cc 100644 --- a/ui/src/styles/layout.css +++ b/ui/src/styles/layout.css @@ -1505,6 +1505,22 @@ html.openclaw-native-macos font-size: 11px; } +.sidebar-session-tree__show-more { + min-height: 28px; + padding: 6px 8px; + border: 0; + background: transparent; + color: var(--muted); + font-size: 11px; + font-weight: 400; + text-align: left; +} + +.sidebar-session-tree__show-more:hover, +.sidebar-session-tree__show-more:focus-visible { + color: var(--text); +} + .sidebar-recent-sessions__group[data-session-section^="catalog:"] > .sidebar-recent-sessions__list { gap: 8px; } diff --git a/ui/src/test-helpers/app-sidebar-cases/child-sessions-cap.ts b/ui/src/test-helpers/app-sidebar-cases/child-sessions-cap.ts new file mode 100644 index 00000000000..362fb14093d --- /dev/null +++ b/ui/src/test-helpers/app-sidebar-cases/child-sessions-cap.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from "vitest"; +import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import { createGateway, createSessionsHarness, mountSidebar } from "../app-sidebar.ts"; +import { waitForFast } from "../wait-for.ts"; +import "../../components/app-sidebar.ts"; + +describe("AppSidebar child session cap", () => { + it("caps visible children until requested and resets the cap after collapse", async () => { + const gateway = createGateway({} as GatewayBrowserClient); + const childKeys = Array.from( + { length: 6 }, + (_, index) => `agent:main:subagent:child-${index + 1}`, + ); + const harness = createSessionsHarness("main", ["agent:main:parent"]); + harness.list.mockResolvedValue({ + ts: 100_000, + path: "", + count: childKeys.length, + defaults: { modelProvider: null, model: null, contextTokens: null }, + sessions: childKeys.map((key, index) => ({ + key, + spawnedBy: "agent:main:parent", + kind: "direct" as const, + label: `Subagent: Child ${index + 1}`, + updatedAt: index + 1, + })), + }); + const { sidebar } = await mountSidebar(gateway, harness.sessions); + harness.publishList({ + result: { + ts: 2, + path: "", + count: 1, + defaults: { modelProvider: null, model: null, contextTokens: null }, + sessions: [ + { + key: "agent:main:parent", + kind: "direct", + label: "Parent task", + updatedAt: 1, + childSessions: childKeys, + }, + ], + }, + }); + await sidebar.updateComplete; + + const toggle = sidebar.querySelector("[data-child-session-toggle]"); + toggle?.click(); + await waitForFast(() => + expect(sidebar.querySelectorAll(".sidebar-recent-session--child")).toHaveLength(4), + ); + const showMore = sidebar.querySelector("[data-show-more-children]"); + expect(showMore?.textContent?.trim()).toBe("Show 2 more"); + expect(showMore?.getAttribute("aria-label")).toBe("Show 2 more"); + expect(sidebar.textContent).not.toContain("Subagent:"); + + showMore?.click(); + await waitForFast(() => + expect(sidebar.querySelectorAll(".sidebar-recent-session--child")).toHaveLength(6), + ); + expect(sidebar.querySelector("[data-show-more-children]")).toBeNull(); + + toggle?.click(); + await sidebar.updateComplete; + toggle?.click(); + await waitForFast(() => + expect(sidebar.querySelectorAll(".sidebar-recent-session--child")).toHaveLength(4), + ); + expect(sidebar.querySelector("[data-show-more-children]")?.textContent).toContain( + "Show 2 more", + ); + }); + + it("keeps live children visible past the cap", async () => { + const gateway = createGateway({} as GatewayBrowserClient); + const childKeys = Array.from( + { length: 6 }, + (_, index) => `agent:main:subagent:child-${index + 1}`, + ); + const harness = createSessionsHarness("main", ["agent:main:parent"]); + harness.list.mockResolvedValue({ + ts: 100_000, + path: "", + count: childKeys.length, + defaults: { modelProvider: null, model: null, contextTokens: null }, + sessions: [ + ...childKeys.map((key, index) => ({ + key, + spawnedBy: "agent:main:parent", + kind: "direct" as const, + label: `Subagent: Child ${index + 1}`, + updatedAt: index + 1, + })), + // Quiet child beyond the cap with a RUNNING grandchild: the branch + // must bypass the cap via the transitive runningChildCount. + { + key: "agent:main:subagent:grandchild", + spawnedBy: "agent:main:subagent:child-6", + kind: "direct" as const, + label: "Subagent: Grandchild run", + updatedAt: 10, + status: "running" as const, + hasActiveRun: true, + }, + ], + }); + const { sidebar } = await mountSidebar(gateway, harness.sessions); + harness.publishList({ + result: { + ts: 2, + path: "", + count: 1, + defaults: { modelProvider: null, model: null, contextTokens: null }, + sessions: [ + { + key: "agent:main:parent", + kind: "direct", + label: "Parent task", + updatedAt: 1, + childSessions: childKeys, + }, + ], + }, + }); + await sidebar.updateComplete; + + sidebar + .querySelector('[data-child-session-toggle="agent:main:parent"]') + ?.click(); + await waitForFast(() => + expect( + sidebar.querySelector('[data-session-key="agent:main:subagent:child-6"]'), + ).not.toBeNull(), + ); + expect(sidebar.querySelector('[data-session-key="agent:main:subagent:child-5"]')).toBeNull(); + expect(sidebar.querySelector("[data-show-more-children]")?.textContent?.trim()).toBe( + "Show 1 more", + ); + }); +}); From 31314920ced79a9fca0612709743140e11473a99 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 11:57:36 +0100 Subject: [PATCH 39/76] improve(ui): plugins hub hierarchy, settings section spacing, and grid-aligned secret inputs (#110482) * improve(ui): align settings section headers, dedupe plugin row status pills, clean hub tab strip * improve(ui): hub tabs as page nav, one-row plugins toolbar, honest section spacing, inset secret-input reveal --- ui/src/components/plugins-hub-tabs.ts | 8 +-- ui/src/components/sessions-hub-tabs.ts | 6 +- ui/src/components/settings-ui.ts | 39 +++++++++++++ ui/src/pages/connection/view.ts | 30 +--------- ui/src/pages/plugins/view.ts | 34 +++++++---- ui/src/styles/plugins.css | 69 +++++++++++++++++++--- ui/src/styles/settings.css | 79 +++++++++++++++++++++++--- 7 files changed, 205 insertions(+), 60 deletions(-) diff --git a/ui/src/components/plugins-hub-tabs.ts b/ui/src/components/plugins-hub-tabs.ts index 9689a4bc225..1c1ab451f7f 100644 --- a/ui/src/components/plugins-hub-tabs.ts +++ b/ui/src/components/plugins-hub-tabs.ts @@ -72,13 +72,13 @@ function reclaimFocus(tab: PluginsHubTab, element: Element | undefined) { /** * Every hub page marks its main content container with * id="plugins-hub-panel" so aria-controls stays valid on each route. - * Styled through the settings design language's segmented control - * (ui/src/styles/settings.css) while keeping tablist semantics. + * Styled as page-level navigation (.hub-tabs in ui/src/styles/plugins.css), + * deliberately distinct from segmented filter pills, keeping tablist semantics. */ export function renderPluginsHubTabs(props: PluginsHubTabsProps) { return html` { // Trusted pointer clicks carry a click count. Keyboard and AT diff --git a/ui/src/components/sessions-hub-tabs.ts b/ui/src/components/sessions-hub-tabs.ts index 0e147730144..40d44f2e749 100644 --- a/ui/src/components/sessions-hub-tabs.ts +++ b/ui/src/components/sessions-hub-tabs.ts @@ -29,12 +29,12 @@ function hubTabLabel(tab: SessionsHubTab): string { /** * Every hub page marks its main content container with * id="sessions-hub-panel" so aria-controls stays valid on each route. - * Styled through the settings design language's segmented control. + * Styled as page-level navigation (.hub-tabs in ui/src/styles/plugins.css). */ export function renderSessionsHubTabs(props: SessionsHubTabsProps) { return html` ${hubTabLabel(tab)} diff --git a/ui/src/components/settings-ui.ts b/ui/src/components/settings-ui.ts index 986b8e0d8a5..fdb7154c81e 100644 --- a/ui/src/components/settings-ui.ts +++ b/ui/src/components/settings-ui.ts @@ -7,6 +7,7 @@ import "@awesome.me/webawesome/dist/components/radio-group/radio-group.js"; import "@awesome.me/webawesome/dist/components/switch/switch.js"; import { html, nothing, type TemplateResult } from "lit"; import { icons } from "./icons.ts"; +import "./tooltip.ts"; type SettingsStatusKind = "ok" | "warn" | "danger" | "accent" | "muted"; @@ -259,3 +260,41 @@ export function renderSettingsValue(value: unknown, options: { mono?: boolean } export function renderSettingsEmpty(message: unknown): TemplateResult { return html`
${message}
`; } + +/** Secret text input with an inset reveal toggle — one field, no trailing + * button, so secret rows line up with plain input rows in the same group. */ +export function renderSettingsSecretInput(props: { + value: string; + placeholder?: string; + visible: boolean; + showLabel: string; + hideLabel: string; + toggleLabel: string; + onInput: (next: string) => void; + onToggle: () => void; +}): TemplateResult { + return html` + + props.onInput((e.target as HTMLInputElement).value)} + /> + + + + + `; +} diff --git a/ui/src/pages/connection/view.ts b/ui/src/pages/connection/view.ts index 3884f5ee330..ac700147342 100644 --- a/ui/src/pages/connection/view.ts +++ b/ui/src/pages/connection/view.ts @@ -2,11 +2,10 @@ import { html } from "lit"; import type { GatewayHelloOk } from "../../api/gateway.ts"; import { resolveGatewayTokenForUrlEdit, type UiSettings } from "../../app/settings.ts"; -import "../../components/tooltip.ts"; -import { icons } from "../../components/icons.ts"; import { renderSettingsPage, renderSettingsRow, + renderSettingsSecretInput, renderSettingsSection, renderSettingsStatus, renderSettingsValue, @@ -43,31 +42,8 @@ function renderSecretRow(params: { onInput: (next: string) => void; onToggle: () => void; }) { - return renderSettingsRow({ - title: params.label, - control: html` - params.onInput((e.target as HTMLInputElement).value)} - placeholder=${params.placeholder} - /> - - - - `, - }); + const { label, ...secret } = params; + return renderSettingsRow({ title: label, control: renderSettingsSecretInput(secret) }); } export function renderConnection(props: ConnectionProps) { diff --git a/ui/src/pages/plugins/view.ts b/ui/src/pages/plugins/view.ts index 6b6119dc163..586d35193d6 100644 --- a/ui/src/pages/plugins/view.ts +++ b/ui/src/pages/plugins/view.ts @@ -341,6 +341,12 @@ function stateStatus(plugin: PluginCatalogItem) { return renderSettingsStatus({ kind, label: stateLabel(plugin) }); } +/** Rows pair the status with an Enable/Disable button that already implies the + * healthy states, so only the error status earns a pill next to the actions. */ +function rowStateStatus(plugin: PluginCatalogItem) { + return plugin.state === "error" ? stateStatus(plugin) : nothing; +} + function originLabel(origin: string): string { switch (origin) { case "bundled": @@ -614,7 +620,7 @@ function renderInstalledRow(plugin: PluginCatalogItem, props: PluginsViewProps): ])}
- ${stateStatus(plugin)} ${renderCatalogActions(plugin, props, busy, key)} + ${rowStateStatus(plugin)} ${renderCatalogActions(plugin, props, busy, key)}
${plugin.error ? html`
- ${renderSettingsStatus({ - kind: server.enabled ? "ok" : "muted", - label: server.enabled ? t("pluginsPage.enabled") : t("pluginsPage.disabled"), - })} ${renderToggleButton(props, props.mcpBusy, { enabled: server.enabled, onToggle: (enabled) => props.onMcpToggle(server.name, enabled), @@ -766,7 +768,6 @@ function renderInstalled(props: PluginsViewProps) { const groups = groupInstalledByCategory(plugins); const filtered = Boolean(props.query || props.installedFilter !== "all"); return html` - ${renderInstalledFilter(props)} ${groups.length === 0 ? renderEmpty( filtered ? t("pluginsPage.noInstalledMatchTitle") : t("pluginsPage.noInstalledTitle"), @@ -821,7 +822,7 @@ function renderCatalogRow(plugin: PluginCatalogItem, props: PluginsViewProps): T ${renderMetaLine([plugin.origin ? originLabel(plugin.origin) : nothing])}
- ${plugin.installed ? stateStatus(plugin) : nothing} + ${plugin.installed ? rowStateStatus(plugin) : nothing} ${renderCatalogActions(plugin, props, busy, key)}
${plugin.error @@ -966,7 +967,7 @@ function renderClawHubResult(item: PluginSearchResult, props: PluginsViewProps):
${installed - ? html`${stateStatus(installed)}${renderCatalogActions(installed, props, busy, key)}` + ? html`${rowStateStatus(installed)}${renderCatalogActions(installed, props, busy, key)}` : renderInstallButton(props, busy, key, pkg.displayName, { source: "clawhub", packageName: pkg.name, @@ -1258,6 +1259,14 @@ function renderActivePanel(props: PluginsViewProps) { export function renderPlugins(props: PluginsViewProps) { const canShowCatalog = Boolean(props.result); + const panelState = + props.loading && !canShowCatalog + ? "loading" + : props.error && !canShowCatalog + ? "error" + : !props.connected && !canShowCatalog + ? "offline" + : "content"; return renderSettingsPage( html`
@@ -1273,6 +1282,9 @@ export function renderPlugins(props: PluginsViewProps) { @input=${(event: Event) => props.onQueryChange((event.currentTarget as HTMLInputElement).value)} /> + ${props.activeTab === "installed" && panelState === "content" + ? renderInstalledFilter(props) + : nothing}
`; } diff --git a/ui/src/components/app-sidebar-menus.ts b/ui/src/components/app-sidebar-menus.ts index 841628dad77..dbc1dd4005e 100644 --- a/ui/src/components/app-sidebar-menus.ts +++ b/ui/src/components/app-sidebar-menus.ts @@ -465,7 +465,7 @@ export abstract class AppSidebarMenusElement extends AppSidebarSessionGroupsElem this.createSessionGroup([session]); break; case "toggle-archived": - void this.patchSession(session, { archived: true }); + void this.archiveSessionWithUndo(session); break; case "stop-cloud-worker": void this.stopCloudWorker(session); diff --git a/ui/src/components/app-sidebar-session-list.ts b/ui/src/components/app-sidebar-session-list.ts index e3b2919c8a7..ab872da463f 100644 --- a/ui/src/components/app-sidebar-session-list.ts +++ b/ui/src/components/app-sidebar-session-list.ts @@ -595,6 +595,13 @@ export abstract class AppSidebarSessionListElement extends AppSidebarMenusElemen codingTrailing: html`${this.renderSessionCatalogs(navigationState)}`, codingTrailingPresent: this.sessionCatalogs.length > 0, })} +
`; diff --git a/ui/src/components/app-sidebar-session-mutations.ts b/ui/src/components/app-sidebar-session-mutations.ts index 87979e1d192..ee36adcfede 100644 --- a/ui/src/components/app-sidebar-session-mutations.ts +++ b/ui/src/components/app-sidebar-session-mutations.ts @@ -4,6 +4,7 @@ import { parseAgentSessionKey, resolveUiConfiguredMainKey, } from "../lib/sessions/session-key.ts"; +import { showToast } from "../lib/toast.ts"; import { AppSidebarSessionNavigationElement } from "./app-sidebar-session-navigation.ts"; import type { SidebarRecentSession, @@ -80,6 +81,78 @@ export abstract class AppSidebarSessionMutationsElement extends AppSidebarSessio return result; } + protected async archiveSessionWithUndo(session: SidebarRecentSession) { + const scope = this.beginSessionMutation(); + if (!scope) { + return; + } + const result = await this.patchSession(session, { archived: true }, scope); + if (result !== "completed" || !this.isSessionMutationScopeCurrent(scope)) { + return; + } + showToast({ + message: t("sessionsView.sessionArchived"), + actionLabel: t("common.undo"), + onAction: () => { + void this.restoreArchivedSessions([{ session, pinned: session.pinned }], scope); + }, + }); + } + + private async archiveSessionsWithUndo(rows: readonly SidebarRecentSession[]) { + const scope = this.beginSessionMutation(); + if (!scope) { + return; + } + const archived: Array<{ session: SidebarRecentSession; pinned: boolean }> = []; + for (const session of rows) { + const result = await this.patchSession(session, { archived: true }, scope); + if (result === "stale") { + return; + } + if (result === "completed") { + archived.push({ session, pinned: session.pinned }); + } + } + if (archived.length === 0 || !this.isSessionMutationScopeCurrent(scope)) { + return; + } + showToast({ + message: + archived.length === 1 + ? t("sessionsView.sessionArchived") + : t("sessionsView.sessionsArchived", { count: String(archived.length) }), + actionLabel: t("common.undo"), + onAction: () => void this.restoreArchivedSessions(archived, scope), + }); + } + + private async restoreArchivedSessions( + archived: readonly { session: SidebarRecentSession; pinned: boolean }[], + scope: SidebarSessionMutationScope, + ) { + if (!this.isSessionMutationScopeCurrent(scope)) { + return; + } + let restoredActiveKey: string | null = null; + for (const { session, pinned } of archived) { + const result = await this.patchSession( + session, + { archived: false, ...(pinned ? { pinned: true } : {}) }, + scope, + ); + if (result === "stale") { + return; + } + if (result === "completed" && session.active) { + restoredActiveKey = session.key; + } + } + if (restoredActiveKey && this.isSessionMutationScopeCurrent(scope)) { + this.replaceCurrentSession(restoredActiveKey); + } + } + /** One confirm and one preserved-worktrees alert for the whole selection. */ protected async deleteSessionsBatch(rows: readonly SidebarRecentSession[]) { if (rows.length === 0) { @@ -153,7 +226,7 @@ export abstract class AppSidebarSessionMutationsElement extends AppSidebarSessio this.createSessionGroup(rows); break; case "toggle-archived": - void this.patchSessions(rows, { archived: true }); + void this.archiveSessionsWithUndo(rows); break; case "delete": void this.deleteSessionsBatch(rows); diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index b075894f148..8ad97b84c24 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -20,6 +20,8 @@ export const en: TranslationMap = { staleData: "Showing stale data.", reload: "Reload", reset: "Reset", + restore: "Restore", + undo: "Undo", probe: "Probe", call: "Call", confirm: "Confirm", @@ -528,7 +530,12 @@ export const en: TranslationMap = { sourceFilters: "Session source filters", global: "Global", unknown: "Unknown", - archivedOnly: "Archived only", + sessionState: "Session state", + viewArchived: "View archived", + sessionArchived: "Session archived", + sessionsArchived: "Archived {count} sessions", + deleteAllArchived: "Delete all archived…", + deleteAllArchivedConfirm: "Delete {count} archived sessions and their transcripts?", activeTooltip: "Loads sessions updated in the last {count} minutes.", limitTooltip: "Max sessions to load.", globalTooltip: "Include global sessions.", diff --git a/ui/src/lib/sessions/index.test.ts b/ui/src/lib/sessions/index.test.ts index d48f50d1d67..7beed42de68 100644 --- a/ui/src/lib/sessions/index.test.ts +++ b/ui/src/lib/sessions/index.test.ts @@ -385,14 +385,17 @@ describe("createSessionCapability", () => { deletedSnapshots.push(next.deletedSessions.map((target) => target.key)); }); - await expect(sessions.deleteMany([{ key: keptKey }, { key: deletedKey }])).resolves.toEqual({ - deleted: [deletedKey], - errors: [], - preservedWorktrees: [], - }); + await expect( + sessions.deleteMany([{ key: keptKey }, { key: deletedKey, archivedOnly: true }]), + ).resolves.toEqual({ deleted: [deletedKey], errors: [], preservedWorktrees: [] }); expect(deletedSnapshots.some((keys) => keys.includes(deletedKey))).toBe(true); expect(deletedSnapshots.some((keys) => keys.includes(keptKey))).toBe(false); expect(request).toHaveBeenCalledTimes(3); + expect(request).toHaveBeenCalledWith("sessions.delete", { + key: deletedKey, + deleteTranscript: true, + archivedOnly: true, + }); unsubscribe(); sessions.dispose(); }); diff --git a/ui/src/lib/sessions/index.ts b/ui/src/lib/sessions/index.ts index 9a4842756a5..43fdbe921e9 100644 --- a/ui/src/lib/sessions/index.ts +++ b/ui/src/lib/sessions/index.ts @@ -98,12 +98,14 @@ export type { SessionPatch } from "./patch.ts"; type SessionDeleteOptions = { agentId?: string; deleteTranscript?: boolean; + archivedOnly?: boolean; }; type SessionDeleteTarget = { key: string; agentId?: string; deleteTranscript?: boolean; + archivedOnly?: boolean; }; /** Dirty/unpushed checkouts survive session deletion; callers surface them. */ @@ -361,6 +363,7 @@ function requestSessionDelete( return client.request("sessions.delete", { ...buildSessionRequestParams(key, options.agentId), deleteTranscript: options.deleteTranscript ?? true, + ...(options.archivedOnly === true ? { archivedOnly: true } : {}), }); } diff --git a/ui/src/lib/toast.test.ts b/ui/src/lib/toast.test.ts new file mode 100644 index 00000000000..782a2a530b5 --- /dev/null +++ b/ui/src/lib/toast.test.ts @@ -0,0 +1,56 @@ +/* @vitest-environment jsdom */ + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { showToast } from "./toast.ts"; + +async function mountHost() { + const host = document.createElement("openclaw-toast-host"); + document.body.append(host); + await host.updateComplete; + return host; +} + +afterEach(() => { + document.body.replaceChildren(); + vi.useRealTimers(); +}); + +describe("shared toast", () => { + it("shows and replaces the active toast", async () => { + const host = await mountHost(); + + showToast({ message: "First" }); + await host.updateComplete; + expect(host.querySelector(".app-toast__message")?.textContent).toBe("First"); + + showToast({ message: "Second" }); + await host.updateComplete; + expect(host.querySelectorAll(".app-toast")).toHaveLength(1); + expect(host.querySelector(".app-toast__message")?.textContent).toBe("Second"); + }); + + it("auto-dismisses after the configured duration", async () => { + vi.useFakeTimers(); + const host = await mountHost(); + + showToast({ message: "Temporary", durationMs: 50 }); + await host.updateComplete; + await vi.advanceTimersByTimeAsync(50); + await host.updateComplete; + + expect(host.querySelector(".app-toast")).toBeNull(); + }); + + it("runs its action once and dismisses", async () => { + const host = await mountHost(); + const onAction = vi.fn(); + showToast({ message: "Archived", actionLabel: "Undo", onAction }); + await host.updateComplete; + + host.querySelector(".app-toast__action")?.click(); + await host.updateComplete; + + expect(onAction).toHaveBeenCalledOnce(); + expect(host.querySelector(".app-toast")).toBeNull(); + }); +}); diff --git a/ui/src/lib/toast.ts b/ui/src/lib/toast.ts new file mode 100644 index 00000000000..33d8dc868c5 --- /dev/null +++ b/ui/src/lib/toast.ts @@ -0,0 +1,92 @@ +import { html, nothing } from "lit"; +import { state } from "lit/decorators.js"; +import { t } from "../i18n/index.ts"; +import { OpenClawLightDomContentsElement } from "../lit/openclaw-element.ts"; + +type ToastOptions = { + message: string; + actionLabel?: string; + onAction?: () => void; + durationMs?: number; +}; + +const DEFAULT_TOAST_DURATION_MS = 6_000; + +class OpenClawToastHost extends OpenClawLightDomContentsElement { + @state() private toast: ToastOptions | null = null; + private dismissTimer: ReturnType | null = null; + + override disconnectedCallback() { + this.clearDismissTimer(); + super.disconnectedCallback(); + } + + show(options: ToastOptions) { + this.clearDismissTimer(); + this.toast = options; + this.dismissTimer = globalThis.setTimeout( + () => this.dismiss(), + options.durationMs ?? DEFAULT_TOAST_DURATION_MS, + ); + } + + private clearDismissTimer() { + if (this.dismissTimer !== null) { + globalThis.clearTimeout(this.dismissTimer); + this.dismissTimer = null; + } + } + + private dismiss() { + this.clearDismissTimer(); + this.toast = null; + } + + override render() { + const toast = this.toast; + if (!toast) { + return nothing; + } + return html` +
+ ${toast.message} + ${toast.actionLabel && toast.onAction + ? html` + + ` + : nothing} + +
+ `; + } +} + +export function showToast(options: ToastOptions): void { + document.querySelector("openclaw-toast-host")?.show(options); +} + +if (!customElements.get("openclaw-toast-host")) { + customElements.define("openclaw-toast-host", OpenClawToastHost); +} + +declare global { + interface HTMLElementTagNameMap { + "openclaw-toast-host": OpenClawToastHost; + } +} diff --git a/ui/src/pages/chat/chat-composer.test.ts b/ui/src/pages/chat/chat-composer.test.ts index 1e032132aad..ea423d1d7e4 100644 --- a/ui/src/pages/chat/chat-composer.test.ts +++ b/ui/src/pages/chat/chat-composer.test.ts @@ -81,6 +81,21 @@ afterEach(async () => { }); describe("renderChatComposer controls", () => { + it("renders and invokes an action beside the disabled reason", () => { + const onDisabledAction = vi.fn(); + const { container } = renderComposer({ + canSend: false, + disabledReason: "This session is archived.", + disabledActionLabel: "Restore", + onDisabledAction, + }); + + const reason = container.querySelector(".agent-chat__disabled-reason"); + expect(reason?.textContent).toContain("This session is archived."); + reason?.querySelector("button")?.click(); + expect(onDisabledAction).toHaveBeenCalledOnce(); + }); + it("switches the primary action between voice, send, queue, and stop", () => { const onToggleRealtimeTalk = vi.fn(); let view = renderComposer({ onToggleRealtimeTalk }); diff --git a/ui/src/pages/chat/chat-pane.ts b/ui/src/pages/chat/chat-pane.ts index faaef120675..d5588ef826b 100644 --- a/ui/src/pages/chat/chat-pane.ts +++ b/ui/src/pages/chat/chat-pane.ts @@ -658,6 +658,29 @@ class ChatPane extends OpenClawLightDomElement { }); } + private async restoreArchivedSession(sessionKey: string) { + const scope = this.captureConnectionScope(); + if (!scope || scope.state.sessionKey !== sessionKey) { + return; + } + const agentId = parseAgentSessionKey(sessionKey)?.agentId ?? resolveChatAgentId(scope.state); + let failure: string | null = null; + try { + // The patch can resolve falsy on failure; the capability error explains it. + const patched = await scope.sessions.patch(sessionKey, { archived: false }, { agentId }); + if (!patched) { + failure = scope.sessions.state.error; + } + } catch (error) { + failure = error instanceof Error ? error.message : String(error); + } + if (failure && this.isConnectionScopeCurrent(scope) && scope.state.sessionKey === sessionKey) { + scope.state.lastError = failure; + scope.state.chatError = failure; + scope.state.requestUpdate?.(); + } + } + private setPaneSessionKey(sessionKey: string): string | null { const state = this.state; if (!state) { @@ -2343,6 +2366,12 @@ class ChatPane extends OpenClawLightDomElement { connected: state.connected, canSend: catalogKey ? this.catalogSession?.canContinue === true : !selectedSessionArchived, disabledReason: catalogDisabledReason ?? disabledReason, + disabledActionLabel: + selectedSessionArchived && !catalogDisabledReason ? t("common.restore") : null, + onDisabledAction: + selectedSessionArchived && !catalogDisabledReason + ? () => void this.restoreArchivedSession(state.sessionKey) + : null, error: state.lastError, sessions: state.sessionsResult, sessionHost: { diff --git a/ui/src/pages/chat/chat-view.ts b/ui/src/pages/chat/chat-view.ts index ba5e77b7bad..20b5f7f4d2c 100644 --- a/ui/src/pages/chat/chat-view.ts +++ b/ui/src/pages/chat/chat-view.ts @@ -112,6 +112,8 @@ export type ChatProps = { connected: boolean; canSend: boolean; disabledReason: string | null; + disabledActionLabel?: string | null; + onDisabledAction?: (() => void) | null; error: string | null; sessions: SessionsListResult | null; /** Host context resolving global-alias session keys (scope=global fleets). */ @@ -330,6 +332,8 @@ export function renderChat(props: ChatProps) { connected: props.connected, canSend: props.canSend, disabledReason: props.disabledReason, + disabledActionLabel: props.disabledActionLabel, + onDisabledAction: props.onDisabledAction, sending: props.sending, canAbort: props.canAbort, runStatus: props.runStatus, @@ -443,7 +447,6 @@ export function renderChat(props: ChatProps) { } }} > - ${props.disabledReason ? html`
${props.disabledReason}
` : nothing} ${props.error ? html` ` : nothing} + ${props.disabledReason + ? html` +
+ ${props.disabledReason} + ${props.disabledActionLabel && props.onDisabledAction + ? html` + + ` + : nothing} +
+ ` + : nothing}
${renderChatAttachmentMenu({ ...props, disabled: !canCompose })} diff --git a/ui/src/pages/sessions/sessions-page.test.ts b/ui/src/pages/sessions/sessions-page.test.ts index 3c52fd99a47..003d252bda4 100644 --- a/ui/src/pages/sessions/sessions-page.test.ts +++ b/ui/src/pages/sessions/sessions-page.test.ts @@ -25,6 +25,7 @@ type TestSessionsPage = HTMLElement & { result: SessionsListResult | null; error: string | null; loading: boolean; + showArchived: boolean; selectedKeys: Set; sessionMenu: { key: string; x: number; y: number } | null; sessionMenuTrigger: HTMLElement | null; @@ -40,6 +41,7 @@ type TestSessionsPage = HTMLElement & { loadCheckpoint: (sessionKey: string) => Promise; deleteSelected: () => Promise; deleteSessionFromMenu: (row: GatewaySessionRow) => Promise; + deleteAllArchived: () => Promise; stopCloudWorker: (row: GatewaySessionRow) => Promise; rememberCustomGroup: (name: string) => Promise; openSessionMenu: ( @@ -47,7 +49,8 @@ type TestSessionsPage = HTMLElement & { position: { x: number; y: number }, trigger: HTMLElement | null, ) => void; - patchSession: (key: string, patch: { archived?: boolean }) => Promise; + patchSession: (key: string, patch: { archived?: boolean; pinned?: boolean }) => Promise; + archiveSessionWithUndo: (row: GatewaySessionRow) => Promise; forkSession: (key: string) => Promise; branchCheckpoint: (sessionKey: string, checkpointId: string) => Promise; restoreCheckpoint: (sessionKey: string, checkpointId: string) => Promise; @@ -177,6 +180,7 @@ async function createPage(context: ApplicationContext): Promise { const page = document.createElement("openclaw-sessions-page") as TestSessionsPage; page.context = context; @@ -186,7 +190,7 @@ async function createRenderedPage( result, error: null, expandedSessionKey: null, - showArchived: false, + showArchived, }; document.body.append(page); await page.updateComplete; @@ -199,6 +203,186 @@ afterEach(() => { }); describe("sessions page lifecycle", () => { + it("switches between Active and Archived with the route parameter", async () => { + const { gateway } = createGateway({} as GatewayBrowserClient); + const context = createContext(gateway, createSessions()); + const page = await createRenderedPage(context, { + ts: Date.now(), + path: "", + count: 0, + defaults: { modelProvider: null, model: null, contextTokens: null }, + sessions: [], + }); + + const archived = [ + ...page.querySelectorAll(".sessions-view-segment button"), + ].find((button) => button.textContent?.trim() === "Archived"); + archived?.click(); + await page.updateComplete; + + expect(page.showArchived).toBe(true); + expect(context.navigate).toHaveBeenCalledWith("sessions", { search: "?showArchived=1" }); + expect(archived?.getAttribute("aria-pressed")).toBe("true"); + }); + + it("re-enumerates all archived sessions before bulk deletion", async () => { + // The rendered result holds one archived row; enumeration must find both. + const archivedKeys = ["agent:main:old-1", "agent:main:old-2"]; + const sessions = createSessions({ + list: vi.fn(async () => ({ + count: 2, + sessions: archivedKeys.map((key) => ({ key, archived: true })), + })) as unknown as SessionCapability["list"], + deleteMany: vi.fn(async () => ({ + deleted: archivedKeys, + errors: [], + preservedWorktrees: [], + })), + }); + const { gateway } = createGateway({} as GatewayBrowserClient); + const page = await createRenderedPage( + createContext(gateway, sessions), + { + count: 2, + sessions: [ + { key: archivedKeys[0], archived: true }, + { key: "agent:main:active", archived: false }, + ], + } as SessionsListResult, + true, + ); + const confirm = vi.spyOn(window, "confirm").mockReturnValue(true); + + page.querySelector(".settings-section__actions .danger")?.click(); + await vi.waitFor(() => expect(sessions.deleteMany).toHaveBeenCalledOnce()); + + expect(sessions.list).toHaveBeenCalledWith( + expect.objectContaining({ showArchived: true, limit: 1000 }), + ); + expect(confirm).toHaveBeenCalledWith("Delete 2 archived sessions and their transcripts?"); + expect(sessions.deleteMany).toHaveBeenCalledWith([ + { + key: archivedKeys[0], + agentId: undefined, + deleteTranscript: true, + archivedOnly: true, + }, + { + key: archivedKeys[1], + agentId: undefined, + deleteTranscript: true, + archivedOnly: true, + }, + ]); + }); + + it("aborts delete-all when an enumeration page fails", async () => { + const sessions = createSessions({ + list: vi.fn(async () => null) as unknown as SessionCapability["list"], + deleteMany: vi.fn(async () => ({ deleted: [], errors: [], preservedWorktrees: [] })), + }); + sessions.state.error = "list failed"; + const { gateway } = createGateway({} as GatewayBrowserClient); + const page = await createRenderedPage( + createContext(gateway, sessions), + { + count: 1, + sessions: [{ key: "agent:main:old-1", archived: true }], + } as SessionsListResult, + true, + ); + const confirm = vi.spyOn(window, "confirm").mockReturnValue(true); + + page.querySelector(".settings-section__actions .danger")?.click(); + await vi.waitFor(() => expect(sessions.list).toHaveBeenCalledOnce()); + + expect(confirm).not.toHaveBeenCalled(); + expect(sessions.deleteMany).not.toHaveBeenCalled(); + }); + + it("paginates the archived enumeration until the listing is exhausted", async () => { + const pageOne = ["agent:main:old-1", "agent:main:old-2"]; + const pageTwo = ["agent:main:old-3"]; + const list = vi + .fn() + .mockResolvedValueOnce({ + count: 3, + sessions: pageOne.map((key) => ({ key, archived: true })), + hasMore: true, + nextOffset: 2, + }) + .mockResolvedValueOnce({ + count: 3, + sessions: pageTwo.map((key) => ({ key, archived: true })), + hasMore: false, + nextOffset: null, + }); + const sessions = createSessions({ + list: list as unknown as SessionCapability["list"], + deleteMany: vi.fn(async () => ({ + deleted: [...pageOne, ...pageTwo], + errors: [], + preservedWorktrees: [], + })), + }); + const { gateway } = createGateway({} as GatewayBrowserClient); + const page = await createRenderedPage( + createContext(gateway, sessions), + { + count: 1, + sessions: [{ key: pageOne[0], archived: true }], + } as SessionsListResult, + true, + ); + const confirm = vi.spyOn(window, "confirm").mockReturnValue(true); + + page.querySelector(".settings-section__actions .danger")?.click(); + await vi.waitFor(() => expect(sessions.deleteMany).toHaveBeenCalledOnce()); + + expect(list).toHaveBeenCalledTimes(2); + expect(list).toHaveBeenNthCalledWith(2, expect.objectContaining({ offset: 2 })); + expect(confirm).toHaveBeenCalledWith("Delete 3 archived sessions and their transcripts?"); + expect(sessions.deleteMany).toHaveBeenCalledWith( + [...pageOne, ...pageTwo].map((key) => ({ + key, + agentId: undefined, + deleteTranscript: true, + archivedOnly: true, + })), + ); + }); + + it("offers undo after archiving from the Sessions page", async () => { + const key = "agent:main:pinned"; + const patch = vi.fn(async () => ({ + ok: true as const, + path: "", + key, + entry: { sessionId: key }, + })); + const sessions = createSessions({ patch }); + const mutableGateway = createGateway({} as GatewayBrowserClient); + mutableGateway.emit({ sessionKey: key }); + const page = await createPage(createContext(mutableGateway.gateway, sessions)); + const toast = document.createElement("openclaw-toast-host"); + document.body.append(toast); + await toast.updateComplete; + + await page.archiveSessionWithUndo({ key, pinned: true } as GatewaySessionRow); + await toast.updateComplete; + toast.querySelector(".app-toast__action")?.click(); + await vi.waitFor(() => expect(patch).toHaveBeenCalledTimes(2)); + await vi.waitFor(() => expect(mutableGateway.setSessionKey).toHaveBeenLastCalledWith(key)); + + expect(patch).toHaveBeenNthCalledWith(1, key, { archived: true }, { agentId: undefined }); + expect(patch).toHaveBeenNthCalledWith( + 2, + key, + { archived: false, pinned: true }, + { agentId: undefined }, + ); + }); + it("submits one trimmed bounded transcript search and adopts its status", async () => { const response = deferred(); const request = vi.fn(() => response.promise); diff --git a/ui/src/pages/sessions/sessions-page.ts b/ui/src/pages/sessions/sessions-page.ts index bcf3f16000a..8fa2307e318 100644 --- a/ui/src/pages/sessions/sessions-page.ts +++ b/ui/src/pages/sessions/sessions-page.ts @@ -36,6 +36,7 @@ import { resolveUiConfiguredMainKey, } from "../../lib/sessions/session-key.ts"; import { normalizeOptionalString } from "../../lib/string-coerce.ts"; +import { showToast } from "../../lib/toast.ts"; import { captureSessionToWorkboard } from "../../lib/workboard/index.ts"; import { OpenClawLightDomElement } from "../../lit/openclaw-element.ts"; import { SubscriptionsController } from "../../lit/subscriptions-controller.ts"; @@ -67,6 +68,8 @@ type SessionsPageRequestScope = { client: GatewayBrowserClient; }; +type SessionsPageMutationResult = "completed" | "failed" | "stale"; + class SessionsPage extends OpenClawLightDomElement { @consume({ context: applicationContext, subscribe: true }) private context?: ApplicationContext; @@ -593,6 +596,22 @@ class SessionsPage extends OpenClawLightDomElement { void this.loadSessions(); } + private updateArchivedView(showArchived: boolean) { + const context = this.context; + if (showArchived === this.showArchived || !context) { + return; + } + this.showArchived = showArchived; + this.page = 0; + this.selectedKeys = new Set(); + this.deepLinkSessionKey = null; + // Route navigation refetches (showArchived is in loaderDeps); mask the old + // view's rows until the new result applies via applyRouteData. + this.loading = true; + this.error = null; + context.navigate("sessions", showArchived ? { search: "?showArchived=1" } : undefined); + } + private async deleteSelected() { const keys = [...this.selectedKeys]; if (keys.length === 0 || this.loading || this.sessionMutationPending) { @@ -608,7 +627,10 @@ class SessionsPage extends OpenClawLightDomElement { await this.deleteSessions(keys); } - private async deleteSessions(keys: string[]) { + private async deleteSessions( + keys: string[], + options: { deleteTranscript?: boolean; archivedOnly?: boolean } = {}, + ) { if (keys.length === 0 || this.loading || this.sessionMutationPending) { return; } @@ -622,6 +644,7 @@ class SessionsPage extends OpenClawLightDomElement { keys.map((key) => ({ key, agentId: this.sessionAgentId(key, scope.context), + ...options, })), ); if (!this.isRequestScopeCurrent(scope)) { @@ -690,6 +713,58 @@ class SessionsPage extends OpenClawLightDomElement { } } + private async deleteAllArchived() { + const scope = this.captureRequestScope(); + if (!scope || this.loading || this.sessionMutationPending) { + return; + } + // The rendered list is bounded by the page's limit filter; re-enumerate the + // full archived set so "all archived" means all of them. Any abnormal page + // (failure, non-advancing offset) aborts: deleting a partial enumeration + // would silently violate the "all archived" contract. + const keys: string[] = []; + try { + // One options snapshot for every page: filter edits made while pages load + // must not mix enumeration populations. + const listOptions = this.sessionListOptions(); + let offset = 0; + for (;;) { + const listed = await scope.sessions.list({ ...listOptions, limit: 1000, offset }); + if (!this.isRequestScopeCurrent(scope)) { + return; + } + if (!listed) { + this.error = scope.sessions.state.error; + return; + } + for (const row of listed.sessions) { + if (row.archived === true) { + keys.push(row.key); + } + } + if (listed.hasMore !== true) { + break; + } + if (typeof listed.nextOffset !== "number" || listed.nextOffset <= offset) { + throw new Error("archived session enumeration did not advance"); + } + offset = listed.nextOffset; + } + } catch (error) { + if (this.isRequestScopeCurrent(scope)) { + this.error = String(error); + } + return; + } + if ( + keys.length === 0 || + !window.confirm(t("sessionsView.deleteAllArchivedConfirm", { count: String(keys.length) })) + ) { + return; + } + await this.deleteSessions(keys, { deleteTranscript: true, archivedOnly: true }); + } + private async deleteSessionFromMenu(row: GatewaySessionRow) { const label = normalizeOptionalString(row.label) ?? row.key; if (!window.confirm(t("sessionsView.deleteSessionConfirm", { session: label }))) { @@ -799,21 +874,24 @@ class SessionsPage extends OpenClawLightDomElement { void this.patchSession(row.key, { label: normalizeOptionalString(value) ?? null }); } - private async patchSession(key: string, patch: Parameters[1]) { - const scope = this.captureRequestScope(); + private async patchSession( + key: string, + patch: Parameters[1], + scope: SessionsPageRequestScope | null = this.captureRequestScope(), + ): Promise { if (!scope) { - return; + return "stale"; } try { const patched = await scope.sessions.patch(key, patch, { agentId: this.sessionAgentId(key, scope.context), }); if (!this.isRequestScopeCurrent(scope)) { - return; + return "stale"; } if (!patched) { this.error = scope.sessions.state.error; - return; + return "failed"; } const selectedKeys = new Set(this.selectedKeys); selectedKeys.delete(key); @@ -835,13 +913,47 @@ class SessionsPage extends OpenClawLightDomElement { }), ); } + return "completed"; } catch (error) { if (this.isRequestScopeCurrent(scope)) { this.error = String(error); + return "failed"; } + return "stale"; } } + private async archiveSessionWithUndo(row: GatewaySessionRow) { + const scope = this.captureRequestScope(); + if (!scope) { + return; + } + const wasActive = areUiSessionKeysEquivalent(row.key, scope.gateway.snapshot.sessionKey); + const result = await this.patchSession(row.key, { archived: true }, scope); + if (result !== "completed" || !this.isRequestScopeCurrent(scope)) { + return; + } + showToast({ + message: t("sessionsView.sessionArchived"), + actionLabel: t("common.undo"), + onAction: () => { + void (async () => { + if (!this.isRequestScopeCurrent(scope)) { + return; + } + const restored = await this.patchSession( + row.key, + { archived: false, ...(row.pinned === true ? { pinned: true } : {}) }, + scope, + ); + if (restored === "completed" && wasActive && this.isRequestScopeCurrent(scope)) { + scope.gateway.setSessionKey(row.key); + } + })(); + }, + }); + } + private async forkSession(key: string) { const scope = this.captureRequestScope(); if (!scope) { @@ -1125,7 +1237,11 @@ class SessionsPage extends OpenClawLightDomElement { this.requestNewCategory(row.key); break; case "toggle-archived": - void this.patchSession(row.key, { archived: row.archived !== true }); + if (row.archived === true) { + void this.patchSession(row.key, { archived: false }); + } else { + void this.archiveSessionWithUndo(row); + } break; case "stop-cloud-worker": void this.stopCloudWorker(row); @@ -1231,6 +1347,8 @@ class SessionsPage extends OpenClawLightDomElement { this.page = 0; }, onRefresh: () => void this.loadSessions(), + onArchivedViewChange: (showArchived) => this.updateArchivedView(showArchived), + onDeleteAllArchived: () => void this.deleteAllArchived(), onPatch: (key, patch) => void this.patchSession(key, patch), onToggleSelect: (key) => { const next = new Set(this.selectedKeys); diff --git a/ui/src/pages/sessions/view.test.ts b/ui/src/pages/sessions/view.test.ts index e6f39ed9b19..d478dbb01bf 100644 --- a/ui/src/pages/sessions/view.test.ts +++ b/ui/src/pages/sessions/view.test.ts @@ -70,6 +70,8 @@ function buildProps(result: SessionsListResult): SessionsProps { onPageChange: () => undefined, onPageSizeChange: () => undefined, onRefresh: () => undefined, + onArchivedViewChange: () => undefined, + onDeleteAllArchived: () => undefined, onPatch: () => undefined, onToggleSelect: () => undefined, onSelectPage: () => undefined, @@ -212,33 +214,29 @@ describe("sessions view", () => { expect(onTranscriptSearch).not.toHaveBeenCalled(); }); - it("renders an explicit archived-session toggle", async () => { + it("renders an Active and Archived segment", async () => { const container = document.createElement("div"); - const onFiltersChange = vi.fn(); + const onArchivedViewChange = vi.fn(); render( renderSessions({ ...buildProps(buildMultiResult([])), - onFiltersChange, + onArchivedViewChange, }), container, ); await Promise.resolve(); - const archivedToggle = container.querySelector( - ".session-archive-toggle input", - ) as HTMLInputElement | null; - expect(archivedToggle?.checked).toBe(false); + const buttons = container.querySelectorAll(".sessions-view-segment button"); + expect([...buttons].map((button) => button.textContent?.trim())).toEqual([ + "Active", + "Archived", + ]); + expect(buttons[0]?.getAttribute("aria-pressed")).toBe("true"); + expect(buttons[1]?.getAttribute("aria-pressed")).toBe("false"); - archivedToggle!.checked = true; - archivedToggle!.dispatchEvent(new Event("change", { bubbles: true })); + buttons[1]?.click(); - expect(onFiltersChange).toHaveBeenCalledWith({ - activeMinutes: "", - limit: "120", - includeGlobal: false, - includeUnknown: false, - showArchived: true, - }); + expect(onArchivedViewChange).toHaveBeenCalledWith(true); }); it("groups sessions by channel with section headers and no pagination", async () => { @@ -441,7 +439,6 @@ describe("sessions view", () => { "Max sessions to load.", "Include global sessions.", "Include unknown sessions.", - "Show only archived sessions.", ]); }); @@ -469,7 +466,7 @@ describe("sessions view", () => { const toggleGroup = container.querySelector(".session-filter-toggle-group"); expect(toggleGroup?.getAttribute("role")).toBe("group"); expect(toggleGroup?.getAttribute("aria-label")).toBe("Session source filters"); - expect(toggleGroup?.querySelectorAll(".session-filter-check")).toHaveLength(3); + expect(toggleGroup?.querySelectorAll(".session-filter-check")).toHaveLength(2); expect( Array.from(toggleGroup?.querySelectorAll(".session-filter-check") ?? []).map((toggle) => [ toggle.querySelector("input")?.getAttribute("name"), @@ -481,7 +478,6 @@ describe("sessions view", () => { ["session-filter-check", "session-filter-toggle", "session-filter-check--active"], ], ["includeUnknown", ["session-filter-check", "session-filter-toggle"]], - ["showArchived", ["session-filter-check", "session-filter-toggle", "session-archive-toggle"]], ]); expect(toggleGroup?.querySelector(".session-filter-check__box")).toBeNull(); }); diff --git a/ui/src/pages/sessions/view.ts b/ui/src/pages/sessions/view.ts index 32bfe355464..5af08f7943b 100644 --- a/ui/src/pages/sessions/view.ts +++ b/ui/src/pages/sessions/view.ts @@ -110,6 +110,8 @@ export type SessionsProps = { onPageChange: (page: number) => void; onPageSizeChange: (size: number) => void; onRefresh: () => void; + onArchivedViewChange: (showArchived: boolean) => void; + onDeleteAllArchived: () => void; onPatch: ( key: string, patch: { @@ -1074,6 +1076,7 @@ export function renderSessions(props: SessionsProps) { const emptyBecauseFiltered = rawRows.length === 0 ? hasActiveFilters(props) : filtered.length === 0; const liveCount = rawRows.filter((row) => isSessionRunActive(row)).length; + const archivedCount = rawRows.filter((row) => row.archived === true).length; const sortHeader = ( col: "key" | "kind" | "updated" | "tokens", @@ -1106,6 +1109,17 @@ export function renderSessions(props: SessionsProps) { : nothing} `; const refreshAction = html` + ${props.showArchived + ? html` + + ` + : nothing} @@ -1166,7 +1180,6 @@ function renderSessionsTable(props: SessionsProps, ctx: SessionsTableContext) { const limitTooltip = t("sessionsView.limitTooltip"); const globalTooltip = t("sessionsView.globalTooltip"); const unknownTooltip = t("sessionsView.unknownTooltip"); - const showArchivedTooltip = t("sessionsView.archivedOnlyTooltip"); return html`
- props.onFiltersChange({ - activeMinutes: props.activeMinutes, - limit: props.limit, - includeGlobal: props.includeGlobal, - includeUnknown: props.includeUnknown, - showArchived: checked, - }), - })} +
+ + +
-
-
-

${t("mcpPage.configuredServers")}

-
-

- ${t("mcpPage.runtimeHint")} - ${t("mcpPage.manageServersLink")} -

-
- ${rows.length - ? rows.map((row) => renderServerRow(row)) - : renderSettingsEmpty(t("mcpPage.noServers"))} -
-
+ ${props.editor} diff --git a/ui/src/pages/config/settings-search.ts b/ui/src/pages/config/settings-search.ts index 05e15aa1765..1cddf93e3d9 100644 --- a/ui/src/pages/config/settings-search.ts +++ b/ui/src/pages/config/settings-search.ts @@ -14,6 +14,7 @@ import { AUTOMATION_SECTION_KEYS, COMMUNICATION_SECTION_KEYS, INFRASTRUCTURE_SECTION_KEYS, + MCP_SECTION_KEYS, SECURITY_SECTION_KEYS, } from "./config-sections.ts"; import { @@ -222,6 +223,7 @@ const COMMUNICATION_SECTIONS = new Set(COMMUNICATION_SECTION_KEYS); const APPEARANCE_SECTIONS = new Set(APPEARANCE_SECTION_KEYS); const SECURITY_SECTIONS = new Set(SECURITY_SECTION_KEYS); const AUTOMATION_SECTIONS = new Set(AUTOMATION_SECTION_KEYS); +const MCP_SECTIONS = new Set(MCP_SECTION_KEYS); const INFRASTRUCTURE_SECTIONS = new Set(INFRASTRUCTURE_SECTION_KEYS); const AI_AGENTS_SECTIONS = new Set(AI_AGENTS_SECTION_KEYS); @@ -236,7 +238,7 @@ function resolveStaticSettingsBlock(block: StaticSettingsBlockDescriptor): Stati } function routeForConfigSection(key: string): RouteId { - if (key === "mcp") { + if (MCP_SECTIONS.has(key)) { return "mcp"; } if (COMMUNICATION_SECTIONS.has(key)) { diff --git a/ui/src/pages/plugins/plugins-page.test.ts b/ui/src/pages/plugins/plugins-page.test.ts index dbc72fa9a5e..f424a5b30b4 100644 --- a/ui/src/pages/plugins/plugins-page.test.ts +++ b/ui/src/pages/plugins/plugins-page.test.ts @@ -869,7 +869,7 @@ describe("PluginsPage", () => { addButton?.click(); await page.updateComplete; - const form = page.querySelector(".plugins-mcp-form")!; + const form = page.querySelector(".mcp-server-form")!; form.querySelector('[name="mcp-name"]')!.value = "context7"; form.querySelector('[name="mcp-target"]')!.value = "https://mcp.context7.com/mcp"; @@ -1021,7 +1021,7 @@ describe("PluginsPage", () => { ].find((button) => button.textContent?.includes("Add server")); addButton?.click(); await page.updateComplete; - const form = page.querySelector(".plugins-mcp-form")!; + const form = page.querySelector(".mcp-server-form")!; form.querySelector('[name="mcp-name"]')!.value = "bad name!"; form.querySelector('[name="mcp-target"]')!.value = "https://x.example/mcp"; form.dispatchEvent(new Event("submit", { bubbles: true, cancelable: true })); diff --git a/ui/src/pages/plugins/plugins-page.ts b/ui/src/pages/plugins/plugins-page.ts index a14ba78f940..bda3e45a620 100644 --- a/ui/src/pages/plugins/plugins-page.ts +++ b/ui/src/pages/plugins/plugins-page.ts @@ -1,6 +1,4 @@ import { consume } from "@lit/context"; -import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url"; -import { asNullableRecord as asRecord } from "@openclaw/normalization-core/record-coerce"; import { html, type PropertyValues } from "lit"; import { property, state } from "lit/decorators.js"; import type { GatewayBrowserClient } from "../../api/gateway.ts"; @@ -13,10 +11,22 @@ import { } from "../../app/context.ts"; import { resolveControlUiAuthCandidates } from "../../app/control-ui-auth.ts"; import { hasOperatorAdminAccess } from "../../app/operator-access.ts"; +import type { McpServerForm } from "../../components/mcp-server-form.ts"; import { renderPluginsHubTabs, type PluginsHubTab } from "../../components/plugins-hub-tabs.ts"; import { renderSettingsWorkspace } from "../../components/settings-workspace.ts"; import { t } from "../../i18n/index.ts"; import { resolveEditableSnapshotConfig } from "../../lib/config/index.ts"; +import { + buildAddMcpServerPatch, + buildRemoveMcpServerPatch, + buildToggleMcpServerPatch, + MCP_SERVER_NAME_PATTERN, + parseMcpTarget, + patchMcpServers, + summarizeMcpServers, + type McpServerSummary, + type McpServersPatchBuildResult, +} from "../../lib/config/mcp-servers.ts"; import { installPlugin, loadPluginCatalog, @@ -41,8 +51,6 @@ import { pluginRowKey, renderPlugins, type InstalledFilter, - type McpServerForm, - type McpServerSummary, type PluginRowMessage, type PluginsTab, } from "./view.ts"; @@ -89,47 +97,6 @@ function mutationSuccessMessage( return lines.filter(Boolean).join("\n"); } -/** Cold MCP server summary mirroring the config page's row projection. */ -function summarizeMcpServers(config: Record | null): McpServerSummary[] | null { - if (!config) { - return null; - } - const servers = asRecord(asRecord(config.mcp)?.servers) ?? {}; - return Object.entries(servers) - .map(([name, value]) => { - const server = asRecord(value) ?? {}; - const url = typeof server.url === "string" ? server.url : ""; - // Command only, mirroring the config page: stdio args routinely carry - // tokens and this surface is visible to read-only operators. - const command = typeof server.command === "string" ? server.command : ""; - const launch = url || command || "missing transport"; - return { - name, - enabled: server.enabled !== false, - transport: url ? ("http" as const) : command ? ("stdio" as const) : ("invalid" as const), - target: url ? redactSensitiveUrlLikeString(launch) : launch, - auth: typeof server.auth === "string" ? server.auth : null, - }; - }) - .toSorted((left, right) => left.name.localeCompare(right.name)); -} - -const MCP_SERVER_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/; - -function parseMcpTarget(target: string): Record | null { - if (/^https?:\/\//i.test(target)) { - // The runtime defaults URL-only servers to SSE; modern MCP endpoints are - // streamable HTTP unless the /sse path convention says otherwise. - const transport = /\/sse\/?$/i.test(target.split("?")[0] ?? target) ? "sse" : "streamable-http"; - return { url: target, transport }; - } - const [command, ...args] = target.trim().split(/\s+/u); - if (!command) { - return null; - } - return args.length > 0 ? { command, args } : { command }; -} - class PluginsPage extends OpenClawLightDomElement { @consume({ context: applicationContext, subscribe: true }) private context!: ApplicationContext; @@ -849,15 +816,8 @@ class PluginsPage extends OpenClawLightDomElement { } } - /** - * Apply one mutation to config.mcp.servers through the shared config seam. - * config.patch uses RFC 7396 merge semantics, so `buildPatch` must return a - * minimal fragment (with explicit nulls for deletions), never a full config. - */ private async mutateMcpServers(params: { - buildPatch: ( - servers: Readonly>, - ) => { patch: Record } | { error: string }; + buildPatch: (servers: Readonly>) => McpServersPatchBuildResult; note: string; successText: string; busyKey?: string; @@ -883,24 +843,13 @@ class PluginsPage extends OpenClawLightDomElement { return false; }; try { - await runtimeConfig.ensureLoaded(); - const base = resolveEditableSnapshotConfig(runtimeConfig.state.configSnapshot); - if (!base) { - return fail(t("pluginsPage.mcpConfigUnavailable")); - } - const servers = asRecord(asRecord(base.mcp)?.servers) ?? {}; - const built = params.buildPatch(servers); - if ("error" in built) { - return fail(built.error); - } - const patched = await runtimeConfig.patch({ - raw: { mcp: { servers: built.patch } }, + const result = await patchMcpServers(runtimeConfig, { + buildPatch: params.buildPatch, note: params.note, }); - if (!patched) { - return fail(runtimeConfig.state.lastError ?? t("pluginsPage.mcpConfigUnavailable")); + if (!result.ok) { + return fail(result.error); } - await runtimeConfig.refresh(); this.syncMcpServers(); this.mcpMessage = { kind: "success", text: params.successText }; return true; @@ -917,21 +866,18 @@ class PluginsPage extends OpenClawLightDomElement { private async addMcpServer(form: McpServerForm) { const name = form.name.trim(); if (!MCP_SERVER_NAME_PATTERN.test(name)) { - this.mcpMessage = { kind: "error", text: t("pluginsPage.mcpNameInvalid") }; + this.mcpMessage = { kind: "error", text: t("mcpServers.nameInvalid") }; return; } const config = parseMcpTarget(form.target); if (!config) { - this.mcpMessage = { kind: "error", text: t("pluginsPage.mcpTargetInvalid") }; + this.mcpMessage = { kind: "error", text: t("mcpServers.targetInvalid") }; return; } const added = await this.mutateMcpServers({ - buildPatch: (servers) => - servers[name] - ? { error: t("pluginsPage.mcpNameTaken", { name }) } - : { patch: { [name]: config } }, + buildPatch: (servers) => buildAddMcpServerPatch(servers, name, config), note: `plugins: add MCP server ${name}`, - successText: t("pluginsPage.mcpAddedSuccess", { name }), + successText: t("mcpServers.addedSuccess", { name }), }); if (added) { this.mcpFormOpen = false; @@ -940,13 +886,9 @@ class PluginsPage extends OpenClawLightDomElement { private async toggleMcpServer(name: string, enabled: boolean) { await this.mutateMcpServers({ - buildPatch: (servers) => - servers[name] - ? // Enabling deletes the key so the config keeps its enabled-by-default shape. - { patch: { [name]: { enabled: enabled ? null : false } } } - : { error: t("pluginsPage.mcpMissing", { name }) }, + buildPatch: (servers) => buildToggleMcpServerPatch(servers, name, enabled), note: `plugins: ${enabled ? "enable" : "disable"} MCP server ${name}`, - successText: t(enabled ? "pluginsPage.enabledSuccess" : "pluginsPage.disabledSuccess", { + successText: t(enabled ? "mcpServers.enabledSuccess" : "mcpServers.disabledSuccess", { name, }), }); @@ -954,12 +896,9 @@ class PluginsPage extends OpenClawLightDomElement { private async removeMcpServer(name: string) { await this.mutateMcpServers({ - buildPatch: (servers) => - servers[name] - ? { patch: { [name]: null } } - : { error: t("pluginsPage.mcpMissing", { name }) }, + buildPatch: (servers) => buildRemoveMcpServerPatch(servers, name), note: `plugins: remove MCP server ${name}`, - successText: t("pluginsPage.mcpRemovedSuccess", { name }), + successText: t("mcpServers.removedSuccess", { name }), }); } @@ -980,9 +919,7 @@ class PluginsPage extends OpenClawLightDomElement { : t("pluginsPage.connectorAddedReady", { name: connector.name }); const added = await this.mutateMcpServers({ buildPatch: (servers) => - servers[mcp.serverName] - ? { error: t("pluginsPage.mcpNameTaken", { name: mcp.serverName }) } - : { patch: { [mcp.serverName]: structuredClone(mcp.config) } }, + buildAddMcpServerPatch(servers, mcp.serverName, structuredClone(mcp.config)), note: `plugins: add MCP connector ${mcp.serverName}`, successText, busyKey: rowKey, diff --git a/ui/src/pages/plugins/view.test.ts b/ui/src/pages/plugins/view.test.ts index b93f29da805..9b36198996e 100644 --- a/ui/src/pages/plugins/view.test.ts +++ b/ui/src/pages/plugins/view.test.ts @@ -331,6 +331,9 @@ describe("renderPlugins", () => { transport: "http", target: "https://api.githubcopilot.com/mcp/", auth: "oauth", + toolFilter: false, + parallel: false, + tls: null, }, ], onMcpToggle, @@ -347,7 +350,7 @@ describe("renderPlugins", () => { actionButton(row, "Remove github")?.click(); expect(onMcpRemove).toHaveBeenCalledWith("github"); - const form = container.querySelector(".plugins-mcp-form")!; + const form = container.querySelector(".mcp-server-form")!; form.querySelector('[name="mcp-name"]')!.value = "context7"; form.querySelector('[name="mcp-target"]')!.value = "https://mcp.context7.com/mcp"; @@ -449,7 +452,16 @@ describe("renderPlugins", () => { createProps({ activeTab: "discover", mcpServers: [ - { name: "github", enabled: true, transport: "http", target: "https://x", auth: "oauth" }, + { + name: "github", + enabled: true, + transport: "http", + target: "https://x", + auth: "oauth", + toolFilter: false, + parallel: false, + tls: null, + }, ], }), ); diff --git a/ui/src/pages/plugins/view.ts b/ui/src/pages/plugins/view.ts index 586d35193d6..94d2dbeb59f 100644 --- a/ui/src/pages/plugins/view.ts +++ b/ui/src/pages/plugins/view.ts @@ -7,6 +7,7 @@ import { html, nothing, type TemplateResult } from "lit"; import { live } from "lit/directives/live.js"; import { repeat } from "lit/directives/repeat.js"; import { icons } from "../../components/icons.ts"; +import { renderMcpServerForm, type McpServerForm } from "../../components/mcp-server-form.ts"; import "../../components/modal-dialog.ts"; import "../../components/openclaw-mascot.ts"; import { @@ -17,6 +18,7 @@ import { renderSettingsStatus, } from "../../components/settings-ui.ts"; import { t } from "../../i18n/index.ts"; +import type { McpServerSummary } from "../../lib/config/mcp-servers.ts"; import { EXTERNAL_LINK_TARGET, buildExternalLinkRel } from "../../lib/external-link.ts"; import "../../styles/plugins.css"; import { @@ -48,19 +50,6 @@ export type PluginRowMessage = { acknowledge?: { packageName: string; version?: string }; }; -export type McpServerSummary = { - name: string; - enabled: boolean; - transport: "stdio" | "http" | "invalid"; - target: string; - auth: string | null; -}; - -export type McpServerForm = { - name: string; - target: string; -}; - type PluginsViewProps = { connected: boolean; loading: boolean; @@ -669,12 +658,20 @@ function renderMcpSection(props: PluginsViewProps) { @click=${() => props.onMcpFormToggle(!props.mcpFormOpen)} > - ${t("pluginsPage.mcpAdd")} + ${t("mcpServers.add")} `, }, html` - ${props.mcpFormOpen ? renderMcpForm(props) : nothing} + ${props.mcpFormOpen + ? renderMcpServerForm({ + busy: props.mcpBusy, + disabled: !props.canMutate, + blockedReason: props.mutationBlockedReason, + onSubmit: props.onMcpAdd, + onCancel: () => props.onMcpFormToggle(false), + }) + : nothing} ${props.mcpMessage ? html`

${server.name}

- ${server.target} + + ${server.target || t("mcpServers.missingTransport")} + ${renderMetaLine([ t("pluginsPage.mcp"), server.transport, @@ -715,54 +714,6 @@ function renderMcpRow(server: McpServerSummary, props: PluginsViewProps): Templa `; } -function renderMcpForm(props: PluginsViewProps) { - const submit = (event: Event) => { - event.preventDefault(); - const form = event.currentTarget as HTMLFormElement; - const data = new FormData(form); - const name = data.get("mcp-name"); - const target = data.get("mcp-target"); - props.onMcpAdd({ - name: typeof name === "string" ? name.trim() : "", - target: typeof target === "string" ? target.trim() : "", - }); - }; - return html` -
- - -
- - -
-
- `; -} - function renderInstalled(props: PluginsViewProps) { const plugins = installedPlugins(props.result?.plugins ?? [], props.query, props.installedFilter); const groups = groupInstalledByCategory(plugins); @@ -878,7 +829,7 @@ function renderConnectorRow( ?disabled=${!props.canMutate || busy} @click=${() => props.onAddConnector(connector)} > - ${busy ? t("pluginsPage.mcpAdding") : t("pluginsPage.connectorAdd")} + ${busy ? t("mcpServers.adding") : t("pluginsPage.connectorAdd")} ` : html` diff --git a/ui/src/styles/config.css b/ui/src/styles/config.css index 7df8c43dfd5..ff7ef44dab4 100644 --- a/ui/src/styles/config.css +++ b/ui/src/styles/config.css @@ -700,10 +700,65 @@ font-family: var(--mono); } +/* Shared inline form stays in config.css because the app-wide stylesheet + * serves both the MCP settings page and the Plugins inventory. */ +.mcp-server-form { + display: grid; + grid-template-columns: minmax(140px, 220px) minmax(0, 1fr) auto; + gap: var(--space-2); + align-items: end; + padding: var(--space-3) var(--space-4); + border-bottom: 1px solid color-mix(in srgb, var(--border) 60%, transparent); +} + +.mcp-server-form label { + display: grid; + gap: 5px; + min-width: 0; +} + +.mcp-server-form label span { + color: var(--muted); + font-size: var(--control-ui-text-xs); + font-weight: 580; +} + +.mcp-server-form__actions { + display: flex; + gap: var(--space-2); +} + +.mcp-server-message { + margin: var(--space-3) var(--space-4); + padding: var(--space-2) var(--space-3); + border-radius: var(--radius-md); + background: var(--ok-subtle); + color: var(--ok); + font-size: var(--control-ui-text-xs); + line-height: 1.45; + white-space: pre-wrap; +} + +.mcp-server-message--error { + background: var(--danger-subtle); + color: var(--danger); +} + +.mcp-server-loading { + padding: var(--space-4); + color: var(--muted); + font-size: var(--control-ui-text-sm); +} + @media (max-width: 820px) { .mcp-command-card__grid { grid-template-columns: 1fr; } + + .mcp-server-form { + grid-template-columns: minmax(0, 1fr); + align-items: stretch; + } } /* =========================================== diff --git a/ui/src/styles/plugins.css b/ui/src/styles/plugins.css index 302cb25d01f..fd9598666ce 100644 --- a/ui/src/styles/plugins.css +++ b/ui/src/styles/plugins.css @@ -530,35 +530,6 @@ h3.plugins-subheader { color: var(--danger); } -/* ---------- MCP inline form ---------- */ - -/* Lives inside the MCP group surface; hairline divider instead of a nested card. */ -.plugins-mcp-form { - display: grid; - grid-template-columns: minmax(140px, 220px) minmax(0, 1fr) auto; - gap: var(--space-2); - align-items: end; - padding: var(--space-3) var(--space-4); - border-bottom: 1px solid color-mix(in srgb, var(--border) 60%, transparent); -} - -.plugins-mcp-form label { - display: grid; - gap: 5px; - min-width: 0; -} - -.plugins-mcp-form label span { - color: var(--muted); - font-size: var(--control-ui-text-xs); - font-weight: 580; -} - -.plugins-mcp-form__actions { - display: flex; - gap: var(--space-2); -} - /* ---------- detail overlay ---------- */ .plugins-detail-backdrop { @@ -739,11 +710,6 @@ h3.plugins-subheader { .plugins-content-header { display: flex; } - - .plugins-mcp-form { - grid-template-columns: minmax(0, 1fr); - align-items: stretch; - } } @media (max-width: 560px) { From bc8d0da0924d8d56d1950cfcec8aa48ede7b3bd8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 13:22:48 +0100 Subject: [PATCH 57/76] feat(linux): Quick Chat streamed replies and precise pairing states (#110632) * feat(linux): stream Quick Chat gateway events * feat(linux): render streamed Quick Chat replies * fix(linux): correlate Quick Chat streams by run * style(linux): rustfmt * test(linux): move Quick Chat stream helper tests into the tooling vitest project * test: type the Linux quickchat stream harness for strict checks * test: explicit harness shape for the root test typecheck --- .../src-tauri/src/gateway_device_identity.rs | 2 + apps/linux/src-tauri/src/gateway_ws.rs | 519 ++++++++++++++---- apps/linux/src-tauri/src/quickchat.rs | 5 +- apps/linux/ui/quickchat.css | 104 +++- apps/linux/ui/quickchat.html | 14 + apps/linux/ui/quickchat.js | 284 +++++++++- docs/platforms/linux.md | 13 +- test/linux-quickchat-stream.test.ts | 270 +++++++++ 8 files changed, 1088 insertions(+), 123 deletions(-) create mode 100644 test/linux-quickchat-stream.test.ts diff --git a/apps/linux/src-tauri/src/gateway_device_identity.rs b/apps/linux/src-tauri/src/gateway_device_identity.rs index 0094981f1bb..0657a86e78a 100644 --- a/apps/linux/src-tauri/src/gateway_device_identity.rs +++ b/apps/linux/src-tauri/src/gateway_device_identity.rs @@ -61,6 +61,8 @@ pub(crate) struct GatewayDeviceIdentityStore { identity: GatewayDeviceIdentity, } +// This credential boundary (gateway_device_identity.rs:65-98) intentionally excludes setup-code +// and bootstrapToken redemption; those require a dedicated product UI and remain follow-up work. #[derive(Clone, Eq, PartialEq)] pub(crate) enum GatewayAuth { DeviceToken(String), diff --git a/apps/linux/src-tauri/src/gateway_ws.rs b/apps/linux/src-tauri/src/gateway_ws.rs index cf701a03406..f6da2924e04 100644 --- a/apps/linux/src-tauri/src/gateway_ws.rs +++ b/apps/linux/src-tauri/src/gateway_ws.rs @@ -26,6 +26,7 @@ use tokio_tungstenite::{ use uuid::Uuid; const GATEWAY_STATE_EVENT: &str = "quickchat:gateway-state"; +const CHAT_EVENT: &str = "quickchat:chat-event"; const GATEWAY_DEVICE_IDENTITY_FILE: &str = "quickchat-gateway-device.json"; const AGENTS_CACHE_TTL: Duration = Duration::from_secs(60); const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); @@ -35,6 +36,8 @@ const COMMAND_TIMEOUT: Duration = Duration::from_secs(35); const DRIVER_TICK: Duration = Duration::from_secs(1); const MAX_RECONNECT_DELAY: Duration = Duration::from_secs(30); const PAIRING_REQUIRED_DETAIL_CODE: &str = "PAIRING_REQUIRED"; +const AUTH_TOKEN_MISSING_DETAIL_CODE: &str = "AUTH_TOKEN_MISSING"; +const AUTH_PASSWORD_MISSING_DETAIL_CODE: &str = "AUTH_PASSWORD_MISSING"; const AUTH_DEVICE_TOKEN_MISMATCH_DETAIL_CODE: &str = "AUTH_DEVICE_TOKEN_MISMATCH"; const TLS_PIN_MISMATCH_ERROR: &str = "Gateway TLS certificate fingerprint mismatch"; @@ -212,6 +215,7 @@ struct ChatSendParams { #[derive(Deserialize)] #[serde(rename_all = "camelCase")] struct ChatSendAck { + run_id: String, status: String, #[serde(default)] error: Option, @@ -219,10 +223,19 @@ struct ChatSendAck { message: Option, } -#[derive(Clone, Debug, Eq, PartialEq)] -struct ChatRoutingTarget { - session_key: String, - agent_id: Option, +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ChatRoutingTarget { + pub(crate) session_key: String, + pub(crate) agent_id: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ChatSendResult { + #[serde(flatten)] + pub(crate) target: ChatRoutingTarget, + pub(crate) run_id: String, } enum GatewayRequest { @@ -243,11 +256,66 @@ enum DriverCommand { Reconfigure, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum GatewayConnectionState { + Down = 0, + Up = 1, + PairingRequired = 2, + CredentialRequired = 3, + TlsFailure = 4, +} + +impl GatewayConnectionState { + fn from_u64(value: u64) -> Self { + match value { + 1 => Self::Up, + 2 => Self::PairingRequired, + 3 => Self::CredentialRequired, + 4 => Self::TlsFailure, + _ => Self::Down, + } + } + + fn event_name(self) -> &'static str { + match self { + Self::Down => "down", + Self::Up => "up", + Self::PairingRequired => "pairing-required", + Self::CredentialRequired => "credential-required", + Self::TlsFailure => "tls-failure", + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct ConnectErrorDetails { + code: Option, + device_id: Option, + remediation_hint: Option, + retryable: Option, + pause_reconnect: Option, +} + +impl ConnectErrorDetails { + fn from_value(value: Option<&Value>) -> Self { + let Some(value) = value else { + return Self::default(); + }; + Self { + code: connect_detail_text(value.get("code"), 80), + device_id: connect_detail_text(value.get("deviceId"), 128), + remediation_hint: connect_detail_text(value.get("remediationHint"), 240), + retryable: value.get("retryable").and_then(Value::as_bool), + pause_reconnect: value.get("pauseReconnect").and_then(Value::as_bool), + } + } +} + struct RequestFailure { message: String, disconnect: bool, - detail_code: Option, - pairing_required: bool, + connect_details: ConnectErrorDetails, + connect_state: Option, tls_failure: bool, } @@ -256,8 +324,8 @@ impl RequestFailure { Self { message: message.into(), disconnect: true, - detail_code: None, - pairing_required: false, + connect_details: ConnectErrorDetails::default(), + connect_state: None, tls_failure: false, } } @@ -266,69 +334,38 @@ impl RequestFailure { Self { message: message.into(), disconnect: true, - detail_code: None, - pairing_required: false, + connect_details: ConnectErrorDetails::default(), + connect_state: None, tls_failure: true, } } - fn method_with_detail(message: impl Into, detail_code: Option) -> Self { + fn method_with_details(message: impl Into, details: Option<&Value>) -> Self { Self { message: message.into(), disconnect: false, - detail_code, - pairing_required: false, + connect_details: ConnectErrorDetails::from_value(details), + connect_state: None, tls_failure: false, } } fn classify_connect(mut self, auth: &GatewayAuth) -> Self { - let pairing_required = self.detail_code.as_deref() == Some(PAIRING_REQUIRED_DETAIL_CODE); - let missing_bootstrap_auth = auth.is_none() - && self - .detail_code - .as_deref() - .is_some_and(|code| code.starts_with("AUTH_")); - self.pairing_required = pairing_required || missing_bootstrap_auth; + self.connect_state = + classify_connect_failure(self.connect_details.code.as_deref(), !auth.is_none()); self } } -#[derive(Clone, Copy, Eq, PartialEq)] -enum GatewayConnectionState { - Down = 0, - Up = 1, - PairingRequired = 2, - TlsFailure = 3, -} - -impl GatewayConnectionState { - fn from_u64(value: u64) -> Self { - match value { - 1 => Self::Up, - 2 => Self::PairingRequired, - 3 => Self::TlsFailure, - _ => Self::Down, - } - } - - fn event_name(self) -> &'static str { - match self { - Self::Down => "down", - Self::Up => "up", - Self::PairingRequired => "pairing-required", - Self::TlsFailure => "tls-failure", - } - } -} - struct GatewayClientInner { config: Mutex>, config_generation: AtomicU64, commands: Mutex>>, agents_cache: Mutex>, identity: Mutex>, + connection_notice: Mutex>, connection_state: AtomicU64, + reconnect_paused: AtomicBool, running: AtomicBool, } @@ -346,7 +383,9 @@ impl GatewayClient { commands: Mutex::new(None), agents_cache: Mutex::new(None), identity: Mutex::new(None), + connection_notice: Mutex::new(None), connection_state: AtomicU64::new(GatewayConnectionState::Down as u64), + reconnect_paused: AtomicBool::new(false), running: AtomicBool::new(false), }), } @@ -364,7 +403,8 @@ impl GatewayClient { .lock() .expect("gateway agents cache mutex poisoned") = None; self.inner.config_generation.fetch_add(1, Ordering::SeqCst); - self.set_connection_state(app, GatewayConnectionState::Down); + self.inner.reconnect_paused.store(false, Ordering::SeqCst); + self.set_connection_state(app, GatewayConnectionState::Down, None); if let Some(commands) = self .inner .commands @@ -388,7 +428,8 @@ impl GatewayClient { .lock() .expect("gateway agents cache mutex poisoned") = None; self.inner.config_generation.fetch_add(1, Ordering::SeqCst); - self.set_connection_state(app, GatewayConnectionState::Down); + self.inner.reconnect_paused.store(false, Ordering::SeqCst); + self.set_connection_state(app, GatewayConnectionState::Down, None); if let Some(commands) = self .inner .commands @@ -417,10 +458,16 @@ impl GatewayClient { } pub fn emit_current_state(&self, window: &WebviewWindow) -> Result<(), String> { + let notice = self + .inner + .connection_notice + .lock() + .map_err(|_| "Gateway connection notice is unavailable.".to_string())? + .clone(); window .emit( GATEWAY_STATE_EVENT, - GatewayStateEvent::new(self.connection_state()), + GatewayStateEvent::new(self.connection_state(), notice), ) .map_err(|error| format!("Could not report Gateway connectivity: {error}")) } @@ -456,12 +503,12 @@ impl GatewayClient { scope: &str, main_key: &str, idempotency_key: &str, - ) -> Result<(), String> { + ) -> Result { let target = routing_target(scope, selected_agent_id, main_key); let response = self .request(GatewayRequest::ChatSend(ChatSendParams { - session_key: target.session_key, - agent_id: target.agent_id, + session_key: target.session_key.clone(), + agent_id: target.agent_id.clone(), message, idempotency_key: idempotency_key.to_string(), })) @@ -469,7 +516,26 @@ impl GatewayClient { let GatewayResponse::ChatSend(ack) = response else { return Err("Gateway returned the wrong response for chat.send.".to_string()); }; - classify_chat_ack(&ack) + classify_chat_ack(&ack)?; + Ok(ChatSendResult { + target, + run_id: ack.run_id, + }) + } + + pub fn resume_reconnect(&self) { + if !self.inner.reconnect_paused.load(Ordering::SeqCst) { + return; + } + if let Some(commands) = self + .inner + .commands + .lock() + .expect("gateway command mutex poisoned") + .as_ref() + { + let _ = commands.try_send(DriverCommand::Reconfigure); + } } async fn request(&self, request: GatewayRequest) -> Result { @@ -498,7 +564,8 @@ impl GatewayClient { let mut reconnect_attempt = 0_u32; loop { if app.get_webview_window(QUICKCHAT_LABEL).is_none() { - self.set_connection_state(&app, GatewayConnectionState::Down); + self.inner.reconnect_paused.store(false, Ordering::SeqCst); + self.set_connection_state(&app, GatewayConnectionState::Down, None); tokio::time::sleep(DRIVER_TICK).await; reconnect_attempt = 0; continue; @@ -510,7 +577,8 @@ impl GatewayClient { .expect("gateway config mutex poisoned") .clone(); let Some(config) = config else { - self.set_connection_state(&app, GatewayConnectionState::Down); + self.inner.reconnect_paused.store(false, Ordering::SeqCst); + self.set_connection_state(&app, GatewayConnectionState::Down, None); tokio::time::sleep(DRIVER_TICK).await; continue; }; @@ -522,12 +590,45 @@ impl GatewayClient { .connect_and_serve(&app, &config, generation, &mut receiver) .await; let reached_hello = self.is_connected(); - let disconnected_state = match connection_result.as_ref() { - Err(failure) if failure.pairing_required => GatewayConnectionState::PairingRequired, - Err(failure) if failure.tls_failure => GatewayConnectionState::TlsFailure, - _ => GatewayConnectionState::Down, - }; - self.set_connection_state(&app, disconnected_state); + let failure = connection_result.as_ref().err(); + let disconnected_state = failure + .and_then(|failure| failure.connect_state) + .or_else(|| { + failure + .is_some_and(|failure| failure.tls_failure) + .then_some(GatewayConnectionState::TlsFailure) + }) + .unwrap_or(GatewayConnectionState::Down); + let pause_reconnect = failure + .map(|failure| should_pause_reconnect(&failure.connect_details)) + .unwrap_or(false); + let notice = failure.and_then(|failure| { + connection_notice( + disconnected_state, + &failure.connect_details, + pause_reconnect, + ) + }); + self.inner + .reconnect_paused + .store(pause_reconnect, Ordering::SeqCst); + self.set_connection_state(&app, disconnected_state, notice); + if pause_reconnect { + // Server retry policy is authoritative: explicit pauseReconnect or retryable=false + // waits for a fresh user summon instead of burning the capped backoff loop. + loop { + let Some(command) = receiver.recv().await else { + return; + }; + match command { + DriverCommand::Reconfigure => break, + command => reject_disconnected_command(command), + } + } + self.inner.reconnect_paused.store(false, Ordering::SeqCst); + reconnect_attempt = 0; + continue; + } reconnect_attempt = if reached_hello { 1 } else { @@ -566,7 +667,7 @@ impl GatewayClient { let signed_at_ms = unix_time_ms().map_err(RequestFailure::transport)?; let params = connect_params(&identity, &auth, &nonce, signed_at_ms) .map_err(RequestFailure::transport)?; - let hello = match request_on_socket(&mut socket, "connect", params).await { + let hello = match request_on_socket(app, &mut socket, "connect", params).await { Ok(hello) => hello, Err(failure) => { let failure = failure.classify_connect(&auth); @@ -582,12 +683,12 @@ impl GatewayClient { self.persist_device_token(&config.ws_url, device_token)?; } - let agents = request_agents_list(&mut socket).await?; + let agents = request_agents_list(app, &mut socket).await?; if self.inner.config_generation.load(Ordering::SeqCst) != generation { return Ok(()); } self.cache_agents(agents); - self.set_connection_state(app, GatewayConnectionState::Up); + self.set_connection_state(app, GatewayConnectionState::Up, None); let mut last_gateway_activity = Instant::now(); loop { @@ -604,7 +705,7 @@ impl GatewayClient { match command { DriverCommand::Reconfigure => return Ok(()), DriverCommand::Request { request, reply } => { - let result = perform_request(&mut socket, request).await; + let result = perform_request(app, &mut socket, request).await; last_gateway_activity = Instant::now(); match result { Ok(response) => { @@ -623,7 +724,7 @@ impl GatewayClient { } } incoming = socket.next() => { - handle_idle_message(&mut socket, incoming).await?; + handle_idle_message(app, &mut socket, incoming).await?; last_gateway_activity = Instant::now(); } _ = tokio::time::sleep(DRIVER_TICK) => { @@ -719,7 +820,12 @@ impl GatewayClient { GatewayConnectionState::from_u64(self.inner.connection_state.load(Ordering::SeqCst)) } - fn set_connection_state(&self, app: &AppHandle, state: GatewayConnectionState) { + fn set_connection_state( + &self, + app: &AppHandle, + state: GatewayConnectionState, + notice: Option, + ) { if state != GatewayConnectionState::Up { *self .inner @@ -727,18 +833,31 @@ impl GatewayClient { .lock() .expect("gateway agents cache mutex poisoned") = None; } - if self + let notice_changed = { + let mut current = self + .inner + .connection_notice + .lock() + .expect("gateway connection notice mutex poisoned"); + if *current == notice { + false + } else { + *current = notice.clone(); + true + } + }; + let state_changed = self .inner .connection_state .swap(state as u64, Ordering::SeqCst) - == state as u64 - { + != state as u64; + if !state_changed && !notice_changed { return; } let _ = app.emit_to( QUICKCHAT_LABEL, GATEWAY_STATE_EVENT, - GatewayStateEvent::new(state), + GatewayStateEvent::new(state, notice), ); } } @@ -746,12 +865,15 @@ impl GatewayClient { #[derive(Clone, Serialize)] struct GatewayStateEvent { state: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + notice: Option, } impl GatewayStateEvent { - fn new(state: GatewayConnectionState) -> Self { + fn new(state: GatewayConnectionState, notice: Option) -> Self { Self { state: state.event_name(), + notice, } } } @@ -777,6 +899,75 @@ fn routing_target(scope: &str, selected_agent_id: &str, main_key: &str) -> ChatR } } +fn connect_detail_text(value: Option<&Value>, max_chars: usize) -> Option { + let normalized = value + .and_then(Value::as_str)? + .split_whitespace() + .collect::>() + .join(" "); + if normalized.is_empty() { + return None; + } + Some(normalized.chars().take(max_chars).collect()) +} + +fn classify_connect_failure( + detail_code: Option<&str>, + has_local_credential: bool, +) -> Option { + if detail_code == Some(PAIRING_REQUIRED_DETAIL_CODE) { + return Some(GatewayConnectionState::PairingRequired); + } + let credential_required = !has_local_credential + && detail_code.is_some_and(|code| { + code == AUTH_TOKEN_MISSING_DETAIL_CODE + || code == AUTH_PASSWORD_MISSING_DETAIL_CODE + || (code.starts_with("AUTH_") && code.ends_with("_MISMATCH")) + }); + credential_required.then_some(GatewayConnectionState::CredentialRequired) +} + +fn should_pause_reconnect(details: &ConnectErrorDetails) -> bool { + details.pause_reconnect == Some(true) || details.retryable == Some(false) +} + +fn short_device_id(device_id: &str) -> Option { + let short = device_id + .chars() + .filter(|character| character.is_ascii_alphanumeric()) + .take(8) + .collect::(); + (!short.is_empty()).then_some(short) +} + +fn connection_notice( + state: GatewayConnectionState, + details: &ConnectErrorDetails, + reconnect_paused: bool, +) -> Option { + let fallback = match state { + GatewayConnectionState::PairingRequired => "Approve this device in the dashboard (Nodes)", + GatewayConnectionState::CredentialRequired => { + "Gateway requires a credential — open the dashboard on the gateway host" + } + _ if reconnect_paused => "Gateway connection paused — reopen Quick Chat to retry", + _ => return None, + }; + // The Gateway owns recovery semantics and can give more precise operator guidance than this + // client. Keep only its bounded plain-text hint, then add the safe pairing identifier. + let mut notice = details + .remediation_hint + .clone() + .unwrap_or_else(|| fallback.to_string()); + if state == GatewayConnectionState::PairingRequired { + if let Some(device_id) = details.device_id.as_deref().and_then(short_device_id) { + notice.push_str(" · Device "); + notice.push_str(&device_id); + } + } + Some(notice) +} + fn reconnect_backoff(attempt: u32) -> Duration { let shift = attempt.saturating_sub(1).min(5); Duration::from_secs((1_u64 << shift).min(MAX_RECONNECT_DELAY.as_secs())) @@ -784,7 +975,7 @@ fn reconnect_backoff(attempt: u32) -> Duration { fn should_clear_stored_device_token(failure: &RequestFailure, auth: &GatewayAuth) -> bool { matches!(auth, GatewayAuth::DeviceToken(_)) - && failure.detail_code.as_deref() == Some(AUTH_DEVICE_TOKEN_MISMATCH_DETAIL_CODE) + && failure.connect_details.code.as_deref() == Some(AUTH_DEVICE_TOKEN_MISMATCH_DETAIL_CODE) } fn connect_params( @@ -849,6 +1040,7 @@ async fn wait_for_connect_challenge(socket: &mut GatewaySocket) -> Result Result { match request { - GatewayRequest::AgentsList => request_agents_list(socket) + GatewayRequest::AgentsList => request_agents_list(app, socket) .await .map(GatewayResponse::AgentsList), GatewayRequest::ChatSend(params) => { let params = serde_json::to_value(params).map_err(|error| { RequestFailure::transport(format!("Could not encode chat.send: {error}")) })?; - let payload = request_on_socket(socket, "chat.send", params).await?; + let payload = request_on_socket(app, socket, "chat.send", params).await?; serde_json::from_value(payload) .map(GatewayResponse::ChatSend) .map_err(|error| { @@ -915,9 +1106,10 @@ async fn perform_request( } async fn request_agents_list( + app: &AppHandle, socket: &mut GatewaySocket, ) -> Result { - let payload = request_on_socket(socket, "agents.list", json!({})).await?; + let payload = request_on_socket(app, socket, "agents.list", json!({})).await?; serde_json::from_value(payload).map_err(|error| { RequestFailure::transport(format!("Invalid agents.list response: {error}")) }) @@ -1089,6 +1281,7 @@ async fn next_json(socket: &mut GatewaySocket) -> Result } async fn handle_idle_message( + app: &AppHandle, socket: &mut GatewaySocket, incoming: Option>, ) -> Result<(), RequestFailure> { @@ -1098,6 +1291,12 @@ async fn handle_idle_message( RequestFailure::transport(format!("Gateway connection failed: {error}")) })?; match message { + Message::Text(text) => { + if let Ok(value) = serde_json::from_str::(text.as_ref()) { + dispatch_chat_event(app, &value); + } + Ok(()) + } Message::Ping(payload) => socket.send(Message::Pong(payload)).await.map_err(|error| { RequestFailure::transport(format!("Could not answer Gateway ping: {error}")) }), @@ -1106,6 +1305,18 @@ async fn handle_idle_message( } } +fn dispatch_chat_event(app: &AppHandle, frame: &Value) { + if frame.get("type").and_then(Value::as_str) != Some("event") + || frame.get("event").and_then(Value::as_str) != Some("chat") + { + return; + } + if let Some(payload) = frame.get("payload") { + // Payload stays raw so the WebView can mirror Gateway delta assembly without native drift. + let _ = app.emit_to(QUICKCHAT_LABEL, CHAT_EVENT, payload.clone()); + } +} + fn unix_time_ms() -> Result { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -1133,6 +1344,11 @@ mod tests { agent_id: None, } ); + assert_eq!( + serde_json::to_value(routing_target("global", "work", "main")) + .expect("serialized routing target"), + json!({ "sessionKey": "global", "agentId": "work" }) + ); } #[test] @@ -1169,6 +1385,7 @@ mod tests { fn chat_ack_acceptance_is_explicit() { for status in ["ok", "started", "in_flight"] { assert!(classify_chat_ack(&ChatSendAck { + run_id: "run-1".to_string(), status: status.to_string(), error: None, message: None, @@ -1177,6 +1394,7 @@ mod tests { } for status in ["error", "timeout", "queued"] { assert!(classify_chat_ack(&ChatSendAck { + run_id: "run-1".to_string(), status: status.to_string(), error: Some(json!({ "message": "not accepted" })), message: None, @@ -1291,33 +1509,122 @@ mod tests { } #[test] - fn pairing_failures_map_to_the_pairing_strip_state() { - let pending = RequestFailure::method_with_detail( - "pairing required", - Some(PAIRING_REQUIRED_DETAIL_CODE.to_string()), - ) - .classify_connect(&GatewayAuth::SharedToken("bootstrap".to_string())); - assert!(pending.pairing_required); + fn connect_classification_separates_pairing_and_missing_credentials() { + assert_eq!( + classify_connect_failure(Some(PAIRING_REQUIRED_DETAIL_CODE), true), + Some(GatewayConnectionState::PairingRequired) + ); + assert_eq!( + classify_connect_failure(Some(AUTH_TOKEN_MISSING_DETAIL_CODE), false), + Some(GatewayConnectionState::CredentialRequired) + ); + assert_eq!( + classify_connect_failure(Some("AUTH_TOKEN_MISMATCH"), false), + Some(GatewayConnectionState::CredentialRequired) + ); + assert_eq!( + classify_connect_failure(Some("AUTH_TOKEN_MISMATCH"), true), + None + ); + assert_eq!( + GatewayConnectionState::CredentialRequired.event_name(), + "credential-required" + ); - let missing_bootstrap = RequestFailure::method_with_detail( - "token missing", - Some("AUTH_TOKEN_MISSING".to_string()), - ) - .classify_connect(&GatewayAuth::None); - assert!(missing_bootstrap.pairing_required); + let pairing_details = json!({ "code": PAIRING_REQUIRED_DETAIL_CODE }); + let pending = + RequestFailure::method_with_details("pairing required", Some(&pairing_details)) + .classify_connect(&GatewayAuth::SharedToken("bootstrap".to_string())); + assert_eq!( + pending.connect_state, + Some(GatewayConnectionState::PairingRequired) + ); + + let missing_details = json!({ "code": AUTH_TOKEN_MISSING_DETAIL_CODE }); + let missing_auth_failure = + RequestFailure::method_with_details("token missing", Some(&missing_details)) + .classify_connect(&GatewayAuth::None); + assert_eq!( + missing_auth_failure.connect_state, + Some(GatewayConnectionState::CredentialRequired) + ); + + let mismatch_details = json!({ "code": "AUTH_TOKEN_MISMATCH" }); + let mismatch_without_auth = + RequestFailure::method_with_details("token mismatch", Some(&mismatch_details)) + .classify_connect(&GatewayAuth::None); + assert_eq!( + mismatch_without_auth.connect_state, + Some(GatewayConnectionState::CredentialRequired) + ); + let mismatch_with_auth = + RequestFailure::method_with_details("token mismatch", Some(&mismatch_details)) + .classify_connect(&GatewayAuth::SharedToken("configured".to_string())); + assert_eq!(mismatch_with_auth.connect_state, None); - let stale_device_auth = RequestFailure::method_with_detail( + let stale_device_details = json!({ "code": AUTH_DEVICE_TOKEN_MISMATCH_DETAIL_CODE }); + let stale_device_auth = RequestFailure::method_with_details( "device token mismatch", - Some(AUTH_DEVICE_TOKEN_MISMATCH_DETAIL_CODE.to_string()), + Some(&stale_device_details), ) .classify_connect(&GatewayAuth::DeviceToken("stale".to_string())); - assert!(!stale_device_auth.pairing_required); + assert_eq!(stale_device_auth.connect_state, None); assert!(should_clear_stored_device_token( &stale_device_auth, &GatewayAuth::DeviceToken("stale".to_string()) )); } + #[test] + fn reconnect_pause_requires_explicit_server_policy() { + let pause_details = json!({ "pauseReconnect": true }); + let paused = RequestFailure::method_with_details("pause", Some(&pause_details)); + assert!(should_pause_reconnect(&paused.connect_details)); + + let terminal_details = json!({ "retryable": false }); + let terminal = RequestFailure::method_with_details("terminal", Some(&terminal_details)); + assert!(should_pause_reconnect(&terminal.connect_details)); + + let retry_details = json!({ "retryable": true, "pauseReconnect": false }); + let retry = RequestFailure::method_with_details("retry", Some(&retry_details)); + assert!(!should_pause_reconnect(&retry.connect_details)); + assert!(!should_pause_reconnect( + &RequestFailure::transport("transport").connect_details + )); + } + + #[test] + fn connection_notices_prefer_server_guidance_and_shorten_device_ids() { + let details = ConnectErrorDetails::from_value(Some(&json!({ + "remediationHint": "Use the Nodes approval queue.", + "deviceId": "abcdef1234567890" + }))); + assert_eq!( + connection_notice(GatewayConnectionState::PairingRequired, &details, true).as_deref(), + Some("Use the Nodes approval queue. · Device abcdef12") + ); + assert_eq!( + connection_notice( + GatewayConnectionState::CredentialRequired, + &ConnectErrorDetails::default(), + true, + ) + .as_deref(), + Some("Gateway requires a credential — open the dashboard on the gateway host") + ); + assert_eq!( + connection_notice( + GatewayConnectionState::Down, + &ConnectErrorDetails::from_value(Some(&json!({ + "remediationHint": "Replace the configured credential." + }))), + true, + ) + .as_deref(), + Some("Replace the configured credential.") + ); + } + #[test] fn chat_send_frame_matches_gateway_schema() { let params = ChatSendParams { @@ -1344,4 +1651,16 @@ mod tests { }) ); } + + #[test] + fn chat_send_result_flattens_route_and_ack_run_id() { + let result = ChatSendResult { + target: routing_target("global", "work", "main"), + run_id: "run-1".to_string(), + }; + assert_eq!( + serde_json::to_value(result).expect("serialized chat send result"), + json!({ "sessionKey": "global", "agentId": "work", "runId": "run-1" }) + ); + } } diff --git a/apps/linux/src-tauri/src/quickchat.rs b/apps/linux/src-tauri/src/quickchat.rs index 724fb801874..ec3d8e7563b 100644 --- a/apps/linux/src-tauri/src/quickchat.rs +++ b/apps/linux/src-tauri/src/quickchat.rs @@ -1,4 +1,4 @@ -use crate::gateway_ws::{AgentsListResult, GatewayClient}; +use crate::gateway_ws::{AgentsListResult, ChatSendResult, GatewayClient}; use crate::{tray, DesktopState}; use serde::Serialize; use std::fs; @@ -513,6 +513,7 @@ pub fn toggle_quickchat(app: &AppHandle) { fn show_quickchat(app: &AppHandle) -> Result<(), String> { let window = ensure_quickchat_window(app)?; + app.state::().resume_reconnect(); window .set_size(LogicalSize::new(QUICKCHAT_WIDTH, QUICKCHAT_HEIGHT)) .map_err(|error| format!("Could not reset Quick Chat size: {error}"))?; @@ -588,7 +589,7 @@ pub async fn quickchat_send( gateway: State<'_, GatewayClient>, state: State<'_, QuickChatState>, message: String, -) -> Result<(), String> { +) -> Result { require_quickchat_window(&window)?; let message = message.trim().to_string(); if message.is_empty() { diff --git a/apps/linux/ui/quickchat.css b/apps/linux/ui/quickchat.css index b1f06abbd65..21f7d9c2124 100644 --- a/apps/linux/ui/quickchat.css +++ b/apps/linux/ui/quickchat.css @@ -220,11 +220,98 @@ body.shown .composer { opacity 120ms ease; } -.composer.has-error .status { +.composer.has-error > .status, +.reply.has-error .reply-error { max-height: 16px; opacity: 1; } +.reply { + padding: 10px 3px 2px 46px; + border-top: 1px solid rgb(255 255 255 / 7%); + margin-top: 9px; +} + +.reply-header { + display: flex; + min-height: 24px; + margin-bottom: 6px; + align-items: center; + justify-content: space-between; +} + +.reply-agent { + display: inline-flex; + min-width: 0; + padding: 2px 7px 2px 2px; + gap: 6px; + align-items: center; + border: 1px solid rgb(255 255 255 / 8%); + border-radius: 999px; + color: #d7d7da; + background: rgb(255 255 255 / 4%); +} + +.reply-agent .agent-avatar-mini { + flex-basis: 20px; + width: 20px; + height: 20px; + font-size: 10px; +} + +.reply-agent-name { + overflow: hidden; + font-size: 11px; + font-weight: 600; + text-overflow: ellipsis; + white-space: nowrap; +} + +.reply-state { + padding-right: 3px; + color: #7f7f84; + font-size: 10px; + letter-spacing: 0.02em; + opacity: 0; + text-transform: uppercase; + transition: opacity 120ms ease; +} + +.reply.is-terminal .reply-state { + opacity: 1; +} + +.reply-scroll { + max-height: 228px; + padding-right: 5px; + overflow-y: auto; + scrollbar-color: rgb(255 255 255 / 16%) transparent; + scrollbar-width: thin; +} + +.reply-text { + overflow-wrap: anywhere; + color: #ededee; + font: 400 13px/1.5 inherit; + white-space: pre-wrap; +} + +.reply-thinking { + display: inline-block; + color: transparent; + font-size: 12px; + background: linear-gradient(100deg, #777 20%, #d2d2d4 48%, #777 76%); + -webkit-background-clip: text; + background-clip: text; + background-size: 220% 100%; + animation: shimmer 1.35s ease-in-out infinite; +} + +.reply-error { + margin-left: 0; + white-space: normal; +} + .popover { position: absolute; z-index: 2; @@ -356,6 +443,16 @@ body.shown .popover:not([hidden]) { } } +@keyframes shimmer { + from { + background-position: 100% 0; + } + + to { + background-position: -100% 0; + } +} + @media (prefers-reduced-motion: reduce) { *, *::before, @@ -369,4 +466,9 @@ body.shown .popover:not([hidden]) { body.shown .composer { transform: none; } + + .reply-thinking { + color: #8e8e93; + background: none; + } } diff --git a/apps/linux/ui/quickchat.html b/apps/linux/ui/quickchat.html index feb757aade1..b3048cfaa89 100644 --- a/apps/linux/ui/quickchat.html +++ b/apps/linux/ui/quickchat.html @@ -43,6 +43,20 @@
+