diff --git a/docs/platformclaw/web-ingress-runtime.md b/docs/platformclaw/web-ingress-runtime.md index 84d6c1614b..aeb2b3c01a 100644 --- a/docs/platformclaw/web-ingress-runtime.md +++ b/docs/platformclaw/web-ingress-runtime.md @@ -361,13 +361,33 @@ Gateway validates each file against the owning Agent's allowed media roots. Plain-text file claims do not grant access. Browser cookies, Gateway credentials, redirects, and unapproved response headers never cross the bridge. -Images from Basic server execution and assigned VMs open in the image viewer. -PDF, PPT, and PPTX attachments offer **Preview** alongside their original -download. PDF bytes are loaded through the owned media bridge and shown in the -browser's PDF reader. PowerPoint previews convert the authorized attachment -snapshot to PDF; the downloadable PPT or PPTX stays unchanged. Reopening the +Images from Basic server execution and assigned VMs appear inline and open in the +image viewer when selected. PDF, PPT, and PPTX attachments open their preview +automatically when their card is visible, alongside the original download. Closing +a preview keeps that file closed until **Preview** is selected again. Both formats +use the same built-in PDF.js viewer, with page navigation and thumbnail contents, +zoom, **Fit page**, **Fit width**, and **Expand document preview** for a larger view. +PowerPoint previews convert the authorized attachment snapshot to PDF before +rendering; the downloadable PPT or PPTX stays unchanged. Reopening the conversation uses the same saved snapshot and checks the current browser owner. +The document fits the existing inline attachment box. Page or slide navigation and +**Request changes** stay below it, inside that box. Selection is off until +**Request changes** is chosen. Click a part or drag a region to open a small input +beside that selection, inside the preview. Describe the change and choose **Send request**. +**Preview options** contains zoom, fitting, and **Expand document preview**. +The ordinary chat outbox sends the request to the owning conversation, replies to +the source message, and attaches a bounded image crop for region selections. +The original document is referenced rather than reattached. A rejected request +keeps the instructions; an admitted request uses the existing queue and Retry. +Closing the preview clears the selection and unsent instructions. + +The viewer, worker, fonts, and character maps are bundled with the app; previews +need no external viewer service or CDN. The browser receives only the authorized +PDF bytes. PDF JavaScript, forms, and external link actions are not executed. +No Canvas setting or `blob:` frame permission is required. A reverse proxy should +preserve the application's CSP rather than replace it with a static policy. + PDF and PowerPoint previews are limited to 16 MiB, including the converted PDF. PowerPoint conversion requires the existing rootless sandbox Docker service and the configured sandbox image with LibreOffice Impress. The PlatformClaw Jammy diff --git a/docs/reference/rich-output-protocol.md b/docs/reference/rich-output-protocol.md index 580b04981e..5394edf484 100644 --- a/docs/reference/rich-output-protocol.md +++ b/docs/reference/rich-output-protocol.md @@ -33,11 +33,22 @@ Legacy final-reply text may still be normalized for compatibility, but this is n ### Image and document previews -In web chat, select an attached image to open the image viewer. Authorized PDF, -PPT, and PPTX attachments offer **Preview** alongside the original download. -PDF files open in the browser's PDF reader. PowerPoint files are converted to PDF -from the saved attachment snapshot; downloading still returns the original PPT -or PPTX bytes. Previewing does not start another model turn. +In web chat, images appear inline; select one to open the image viewer. Authorized +PDF, PPT, and PPTX attachments open their preview automatically when visible. +Closing a preview keeps that file closed; select **Preview** to reopen it. +PDF and PowerPoint use the same built-in PDF.js viewer inside the attachment box, +fitted to its width, with page navigation below the preview. **Contents** opens +thumbnails; **Preview options** contains zoom, fitting, and an optional expanded view. +Select **Request changes**, then click the part to change or drag a region. A small +input opens beside the selection, inside the same preview. Describe the change and +select **Send request** to send it to the current agent. +The request replies to the original message and includes a crop when a region is +selected; it does not reattach the original document. Closing or changing the +document clears the local selection. The renderer and its assets +ship with the app and do not use an external viewer service. PowerPoint files +are converted to PDF from the saved attachment snapshot; downloading still +returns the original PPT or PPTX bytes. Viewing, navigating, and selecting do not +start another model turn; only submitting the change request sends a chat message. PDF and PowerPoint previews accept files up to 16 MiB. PowerPoint conversion also limits its PDF output to 16 MiB. It requires the rootless sandbox Docker service diff --git a/patches/pdfjs-dist@6.4.299.patch b/patches/pdfjs-dist@6.4.299.patch new file mode 100644 index 0000000000..957ae15d57 --- /dev/null +++ b/patches/pdfjs-dist@6.4.299.patch @@ -0,0 +1,35 @@ +--- a/build/pdf.mjs ++++ b/build/pdf.mjs +@@ -15964,6 +15964,11 @@ + isEditing = false + } = {}) { + function operatorListChanged() { ++ if (intentState.operatorList.error) { ++ intentState.opListReadCapability.reject(intentState.operatorList.error); ++ intentState.renderTasks.delete(opListTask); ++ return; ++ } + if (intentState.operatorList.lastChunk) { + intentState.opListReadCapability.resolve(intentState.operatorList); + intentState.renderTasks.delete(opListTask); +@@ -16133,6 +16138,7 @@ + return; + } + if (intentState.operatorList) { ++ intentState.operatorList.error = reason; + intentState.operatorList.lastChunk = true; + for (const internalRenderTask of intentState.renderTasks) { + internalRenderTask.operatorListChanged(); +@@ -17118,6 +17124,12 @@ + this.task.onError?.(error); + } + operatorListChanged() { ++ if (this.operatorList.error) { ++ if (!this.cancelled) { ++ this.cancel(this.operatorList.error); ++ } ++ return; ++ } + if (!this.graphicsReady) { + this.graphicsReadyCallback ||= this._continueBound; + return; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aaa76a1524..ebbfac36d3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,7 @@ packageExtensionsChecksum: sha256-zZ8fyodhMTumshonC7kktCqTPsiHL3UAyS9vltFAlMo= patchedDependencies: '@azure/msal-node@5.4.2': c2854316529b20e93e91e44a4edb703526a9b2cfed9512baa9d4ac9c6f200679 '@discord/embedded-app-sdk@2.5.0': f48e401bc4364900e83f8fe2a1319182b3ef04cf1aef48cb518c7e3010e9af10 + pdfjs-dist@6.4.299: efdb08019c1e2d062dfae9dd4b86e3bf1b5f6c3ee52f5fe8213d6bc4d4df15e0 zod@3.25.76: 7603e669a671f61a89c898cba1c3eedbb5c529274ac7187ec3394c10a055dec9 zod@4.6.6: 5b448a1a3aafd543cd67bc4520817f995b85324b886be37256af9b4cb15401aa @@ -2556,6 +2557,9 @@ importers: marked: specifier: 18.0.7 version: 18.0.7 + pdfjs-dist: + specifier: 6.4.299 + version: 6.4.299(patch_hash=efdb08019c1e2d062dfae9dd4b86e3bf1b5f6c3ee52f5fe8213d6bc4d4df15e0) remend: specifier: 1.3.0 version: 1.3.0 @@ -4041,6 +4045,81 @@ packages: resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==} engines: {node: '>=14.0.0'} + '@napi-rs/canvas-android-arm64@1.1.0': + resolution: {integrity: sha512-mKrSmRyhWK4q699Q1xthbZIZi7JBvLtsRNnOZAeFaLPOFT1yo8y82aRC2nNZbXTGJXU7fVDONgGQPr0vo3kerw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/canvas-darwin-arm64@1.1.0': + resolution: {integrity: sha512-iqwST4g0oH+x8VCpBIYh+MYokv/mVvkM+v9oJGB0d9A3P2qBh42ogJlVyWq9SpPCrAjS7tplMb1D3OIfxl+utw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/canvas-darwin-x64@1.1.0': + resolution: {integrity: sha512-jkeA8eVTxDum03LTvc32YJ28F6KB1U0gYwnWMfOAUQZvWWEh/kvONby4iWGJx3+Tw45UqMGMXjtw2GKNVZDhVg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/canvas-linux-arm-gnueabihf@1.1.0': + resolution: {integrity: sha512-Y4YkKG1ijoqkgUdTEEBNG4k+2BK5bR5jGL6S56YxCXYPwGfUd5nED8/74xyc7z+bpixPasCjSYoP3xYPOIGNxw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/canvas-linux-arm64-gnu@1.1.0': + resolution: {integrity: sha512-dE/XthqYkiWcBVtKFlZbUXwAYfDuQGqDWixznipJNmevLGpprNnTVQZyftm1RZJudkIOn2EkzYZDMMETM08Row==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@napi-rs/canvas-linux-arm64-musl@1.1.0': + resolution: {integrity: sha512-DqZ/CMcHlhU56pPjv9AP0h3U7MFQ5LzC1HzMyRMotL+PFsJmZJA/ziApPYVyyz78DaVLhI8t3So3ztqtijbYpg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@napi-rs/canvas-linux-riscv64-gnu@1.1.0': + resolution: {integrity: sha512-kJ7m5n2VffxWXM5QGa5lfq8ZN4LdOJEoz3aAJAmIqYvKiZw1gRdXqpucnRXsF6D9nLYiiIlevaAtAclR7c5ilg==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@napi-rs/canvas-linux-x64-gnu@1.1.0': + resolution: {integrity: sha512-uQ9JuJlRhQdHyttcMWOiwLkfIqkqaB0YFUVe835LUe12THt2b51HXkTItq0fxWT1jdhUPX2ylWZ1walkJyD6GQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@napi-rs/canvas-linux-x64-musl@1.1.0': + resolution: {integrity: sha512-has4CPPAkkCac/LZ13VAw5p+MJLdaKVoyrhumN/A7uJd6960xYkJNDZgUebyxiha82AFzXx6RV+QJg5qGG079g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@napi-rs/canvas-win32-arm64-msvc@1.1.0': + resolution: {integrity: sha512-MJnktLKNijmXqdjTughHDZ9TDLZ9uUm2hvFXqaOqn0+w7lphXFnft6bpSgQAc9U1exc20UGIBlzGu9+kiCj9Ag==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/canvas-win32-x64-msvc@1.1.0': + resolution: {integrity: sha512-08575bFk7iiadxap4h+JxgGut9xDCHAGum6wh0eInN53zEMImkmEHi5MushX2FOmNO7qRNX7BPiDC5ZYBwHeEg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/canvas@1.1.0': + resolution: {integrity: sha512-k+ph/D8H9GS8ZZ48okvHrKeJynkqAA1kMUZRuzRWNlQUDHXngq7oRaWMZBdmSfd9ZhVy9bwNKei/7wMboI/uMg==} + engines: {node: '>= 10'} + '@napi-rs/wasm-runtime@1.1.6': resolution: {integrity: sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==} peerDependencies: @@ -8293,6 +8372,10 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pdfjs-dist@6.4.299: + resolution: {integrity: sha512-AVl138zALtfaAPvADulE0PZThbYzCBS79nL4pOSL/6Sm/4AH5A21BD9VHt97OlCuzJuCpmeZtAtkinisF4Vb1g==} + engines: {node: '>=22.13.0 || >=24'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -11439,6 +11522,54 @@ snapshots: '@mozilla/readability@0.6.0': {} + '@napi-rs/canvas-android-arm64@1.1.0': + optional: true + + '@napi-rs/canvas-darwin-arm64@1.1.0': + optional: true + + '@napi-rs/canvas-darwin-x64@1.1.0': + optional: true + + '@napi-rs/canvas-linux-arm-gnueabihf@1.1.0': + optional: true + + '@napi-rs/canvas-linux-arm64-gnu@1.1.0': + optional: true + + '@napi-rs/canvas-linux-arm64-musl@1.1.0': + optional: true + + '@napi-rs/canvas-linux-riscv64-gnu@1.1.0': + optional: true + + '@napi-rs/canvas-linux-x64-gnu@1.1.0': + optional: true + + '@napi-rs/canvas-linux-x64-musl@1.1.0': + optional: true + + '@napi-rs/canvas-win32-arm64-msvc@1.1.0': + optional: true + + '@napi-rs/canvas-win32-x64-msvc@1.1.0': + optional: true + + '@napi-rs/canvas@1.1.0': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 1.1.0 + '@napi-rs/canvas-darwin-arm64': 1.1.0 + '@napi-rs/canvas-darwin-x64': 1.1.0 + '@napi-rs/canvas-linux-arm-gnueabihf': 1.1.0 + '@napi-rs/canvas-linux-arm64-gnu': 1.1.0 + '@napi-rs/canvas-linux-arm64-musl': 1.1.0 + '@napi-rs/canvas-linux-riscv64-gnu': 1.1.0 + '@napi-rs/canvas-linux-x64-gnu': 1.1.0 + '@napi-rs/canvas-linux-x64-musl': 1.1.0 + '@napi-rs/canvas-win32-arm64-msvc': 1.1.0 + '@napi-rs/canvas-win32-x64-msvc': 1.1.0 + optional: true + '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)': dependencies: '@emnapi/core': 1.11.1 @@ -15893,6 +16024,10 @@ snapshots: pathe@2.0.3: {} + pdfjs-dist@6.4.299(patch_hash=efdb08019c1e2d062dfae9dd4b86e3bf1b5f6c3ee52f5fe8213d6bc4d4df15e0): + optionalDependencies: + '@napi-rs/canvas': 1.1.0 + picocolors@1.1.1: {} picomatch@2.3.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0efe438c4e..9de1e9bd46 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -196,5 +196,6 @@ packageExtensions: patchedDependencies: "@azure/msal-node@5.4.2": patches/@azure__msal-node@5.4.2.patch "@discord/embedded-app-sdk@2.5.0": patches/@discord__embedded-app-sdk@2.5.0.patch + pdfjs-dist@6.4.299: patches/pdfjs-dist@6.4.299.patch zod@3.25.76: patches/zod@3.25.76.patch zod@4.6.6: patches/zod@4.6.6.patch diff --git a/scripts/check-control-ui-performance.d.mts b/scripts/check-control-ui-performance.d.mts index 1a0e431f00..2f5451c3c2 100644 --- a/scripts/check-control-ui-performance.d.mts +++ b/scripts/check-control-ui-performance.d.mts @@ -21,6 +21,8 @@ export type ControlUiPerformanceMetrics = { assets: ControlUiAssetMetrics[]; }; total: { js: ControlUiAssetSummary; css: ControlUiAssetSummary }; + workers: { js: ControlUiAssetSummary; largestJs?: ControlUiAssetMetrics }; + /** Ordinary/main-thread JavaScript retains its independent ceiling. */ largest: { js: ControlUiAssetMetrics; css: ControlUiAssetMetrics }; }; @@ -30,6 +32,7 @@ export type ControlUiPerformanceBudgets = { startupJsGzipBytes: number; startupCssGzipBytes: number; largestJsGzipBytes: number; + largestWorkerJsGzipBytes: number; largestCssGzipBytes: number; }; diff --git a/scripts/check-control-ui-performance.mjs b/scripts/check-control-ui-performance.mjs index 6b872ba6bc..0ae4a82253 100644 --- a/scripts/check-control-ui-performance.mjs +++ b/scripts/check-control-ui-performance.mjs @@ -6,6 +6,8 @@ import process from "node:process"; import { fileURLToPath } from "node:url"; const KIB = 1024; +// Vite owns this directory for every worker entry and worker-only chunk. +const WORKER_ASSET_PREFIX = "assets/workers/"; const STARTUP_JS_BASELINE_RATCHET_BYTES = 4096; const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); const DEFAULT_STARTUP_BUDGET_BASELINE_PATH = path.resolve( @@ -31,6 +33,9 @@ export const CONTROL_UI_PERFORMANCE_BUDGETS = Object.freeze({ // sidebar zone styling; headroom over the ~36.5 KiB post-diet baseline. startupCssGzipBytes: 45 * KIB, largestJsGzipBytes: 215 * KIB, + // Explicitly approved for the isolated, lazy PDF renderer. This does not + // increase the main-thread or startup ceilings, or omit worker transport bytes. + largestWorkerJsGzipBytes: 360 * KIB, largestCssGzipBytes: 45 * KIB, }); @@ -41,7 +46,7 @@ function controlUiAssetPathFromUrl(value) { return null; } const assetPath = normalized.slice(markerIndex); - if (assetPath.includes("../") || !/\.(?:css|js)$/u.test(assetPath)) { + if (assetPath.includes("../")) { return null; } return assetPath; @@ -52,32 +57,51 @@ export function extractControlUiStartupAssetPaths(html) { for (const tag of html.matchAll(/<(?:link|script)\b[^>]*>/giu)) { const attribute = tag[0].match(/\s(?:href|src)\s*=\s*["']([^"']+)["']/iu); const assetPath = attribute ? controlUiAssetPathFromUrl(attribute[1]) : null; - if (assetPath) { + if (assetPath?.startsWith(WORKER_ASSET_PREFIX)) { + throw new Error(`Control UI worker asset must not appear in startup HTML: ${assetPath}`); + } + if (assetPath && /\.(?:css|m?js)$/u.test(assetPath)) { assets.add(assetPath); } } return [...assets].toSorted((left, right) => left.localeCompare(right)); } -function readAssetMetrics(assetsDir, entry) { - const file = `assets/${entry.name}`; - const sourcePath = path.join(assetsDir, entry.name); +function readAssetMetrics(assetsDir, relativeFile) { + const file = `assets/${relativeFile}`; + const sourcePath = path.join(assetsDir, relativeFile); const gzipPath = `${sourcePath}.gz`; const brotliPath = `${sourcePath}.br`; for (const sidecarPath of [gzipPath, brotliPath]) { if (!fs.existsSync(sidecarPath)) { - throw new Error(`Control UI performance check missing ${path.basename(sidecarPath)}`); + throw new Error( + `Control UI performance check missing ${relativeFile}${path.extname(sidecarPath)}`, + ); } } return { file, - type: entry.name.endsWith(".js") ? "js" : "css", + type: relativeFile.endsWith(".css") ? "css" : "js", rawBytes: fs.statSync(sourcePath).size, gzipBytes: fs.statSync(gzipPath).size, brotliBytes: fs.statSync(brotliPath).size, }; } +function collectAssetMetrics(assetsDir, directory = "") { + return fs + .readdirSync(path.join(assetsDir, directory), { withFileTypes: true }) + .flatMap((entry) => { + const relativeFile = path.posix.join(directory, entry.name); + if (entry.isDirectory()) { + return collectAssetMetrics(assetsDir, relativeFile); + } + return entry.isFile() && /\.(?:css|m?js)$/u.test(entry.name) + ? [readAssetMetrics(assetsDir, relativeFile)] + : []; + }); +} + function summarizeAssets(assets) { return assets.reduce( (summary, asset) => ({ @@ -99,10 +123,7 @@ function largestAsset(assets) { export function collectControlUiPerformanceMetrics(distDir) { const assetsDir = path.join(distDir, "assets"); const html = fs.readFileSync(path.join(distDir, "index.html"), "utf8"); - const assets = fs - .readdirSync(assetsDir, { withFileTypes: true }) - .filter((entry) => entry.isFile() && /\.(?:css|js)$/u.test(entry.name)) - .map((entry) => readAssetMetrics(assetsDir, entry)); + const assets = collectAssetMetrics(assetsDir); const assetsByFile = new Map(assets.map((asset) => [asset.file, asset])); const startup = extractControlUiStartupAssetPaths(html).map((file) => { const asset = assetsByFile.get(file); @@ -112,8 +133,10 @@ export function collectControlUiPerformanceMetrics(distDir) { return asset; }); const jsAssets = assets.filter((asset) => asset.type === "js"); + const workerJsAssets = jsAssets.filter((asset) => asset.file.startsWith(WORKER_ASSET_PREFIX)); + const mainJsAssets = jsAssets.filter((asset) => !asset.file.startsWith(WORKER_ASSET_PREFIX)); const cssAssets = assets.filter((asset) => asset.type === "css"); - if (jsAssets.length === 0 || cssAssets.length === 0 || startup.length === 0) { + if (mainJsAssets.length === 0 || cssAssets.length === 0 || startup.length === 0) { throw new Error("Control UI performance check found an incomplete production bundle"); } return { @@ -127,8 +150,12 @@ export function collectControlUiPerformanceMetrics(distDir) { js: summarizeAssets(jsAssets), css: summarizeAssets(cssAssets), }, + workers: { + js: summarizeAssets(workerJsAssets), + largestJs: largestAsset(workerJsAssets), + }, largest: { - js: largestAsset(jsAssets), + js: largestAsset(mainJsAssets), css: largestAsset(cssAssets), }, }; @@ -148,6 +175,14 @@ export function evaluateControlUiPerformanceBudgets( ["largest JS gzip", metrics.largest.js.gzipBytes, budgets.largestJsGzipBytes, "bytes"], ["largest CSS gzip", metrics.largest.css.gzipBytes, budgets.largestCssGzipBytes, "bytes"], ]; + if (metrics.workers.largestJs) { + checks.push([ + "largest worker JS gzip", + metrics.workers.largestJs.gzipBytes, + budgets.largestWorkerJsGzipBytes, + "bytes", + ]); + } const violations = checks.flatMap(([metric, actual, limit, unit]) => actual > limit ? [{ metric, actual, limit, unit }] : [], ); @@ -221,10 +256,12 @@ export function formatControlUiPerformanceReport( } lines.push( ` startup CSS: ${formatAssetSummary(metrics.startup.css)} (limits: ${formatRequestCount(budgets.startupCssRequests)}, ${formatControlUiPerformanceBytes(budgets.startupCssGzipBytes)} gzip)`, - ` largest JS: ${metrics.largest.js.file}, ${formatControlUiPerformanceBytes(metrics.largest.js.gzipBytes)} gzip (limit: ${formatControlUiPerformanceBytes(budgets.largestJsGzipBytes)})`, + ` largest main-thread JS: ${metrics.largest.js.file}, ${formatControlUiPerformanceBytes(metrics.largest.js.gzipBytes)} gzip (limit: ${formatControlUiPerformanceBytes(budgets.largestJsGzipBytes)})`, ` largest CSS: ${metrics.largest.css.file}, ${formatControlUiPerformanceBytes(metrics.largest.css.gzipBytes)} gzip (limit: ${formatControlUiPerformanceBytes(budgets.largestCssGzipBytes)})`, ` all JS: ${formatAssetSummary(metrics.total.js)}`, ` all CSS: ${formatAssetSummary(metrics.total.css)}`, + ` worker JS: ${formatAssetSummary(metrics.workers.js)}`, + ` largest worker JS: ${metrics.workers.largestJs ? `${metrics.workers.largestJs.file}, ${formatControlUiPerformanceBytes(metrics.workers.largestJs.gzipBytes)} gzip` : "none"} (limit: ${formatControlUiPerformanceBytes(budgets.largestWorkerJsGzipBytes)})`, ); if ( startupBudgetBaseline && diff --git a/scripts/check-control-ui-precompressed-assets.mjs b/scripts/check-control-ui-precompressed-assets.mjs index f2fe536c9b..628090fab2 100644 --- a/scripts/check-control-ui-precompressed-assets.mjs +++ b/scripts/check-control-ui-precompressed-assets.mjs @@ -9,7 +9,7 @@ const assetsDir = path.join(repoRoot, "dist", "control-ui", "assets"); const errors = []; let checked = 0; -for (const entry of fs.readdirSync(assetsDir, { withFileTypes: true })) { +for (const entry of fs.readdirSync(assetsDir, { withFileTypes: true, recursive: true })) { if (!entry.isFile()) { continue; } @@ -17,17 +17,18 @@ for (const entry of fs.readdirSync(assetsDir, { withFileTypes: true })) { if (!suffix) { continue; } - const sidecarPath = path.join(assetsDir, entry.name); + const sidecarPath = path.join(entry.parentPath, entry.name); + const relativePath = path.relative(assetsDir, sidecarPath); const sourcePath = sidecarPath.slice(0, -suffix.length); try { const encoded = fs.readFileSync(sidecarPath); const decoded = suffix === ".br" ? brotliDecompressSync(encoded) : gunzipSync(encoded); const source = fs.readFileSync(sourcePath); if (!decoded.equals(source)) { - errors.push(`${entry.name}: decoded bytes differ from ${path.basename(sourcePath)}`); + errors.push(`${relativePath}: decoded bytes differ from ${path.basename(sourcePath)}`); } } catch (error) { - errors.push(`${entry.name}: ${error instanceof Error ? error.message : String(error)}`); + errors.push(`${relativePath}: ${error instanceof Error ? error.message : String(error)}`); } checked += 1; } diff --git a/scripts/check-package-patches.mjs b/scripts/check-package-patches.mjs index e010daf7bf..5bd2d54262 100644 --- a/scripts/check-package-patches.mjs +++ b/scripts/check-package-patches.mjs @@ -2,6 +2,7 @@ // Guards pnpm package patches against unapproved additions. import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -16,12 +17,30 @@ const LEGACY_PATCHED_DEPENDENCIES = new Map([ ["baileys@7.0.0-rc13", "patches/baileys@7.0.0-rc13.patch"], ]); const reviewedSecurityPatches = getReviewedSecurityPatchRegistrations(); +// Approved product repair: propagate PDF.js operator-list stream failures to +// rendering callers. This registration does not remediate or suppress advisories. +const REVIEWED_PRODUCT_PATCHES = [ + { + specifier: "pdfjs-dist@6.4.299", + patchPath: "patches/pdfjs-dist@6.4.299.patch", + sha256: "efdb08019c1e2d062dfae9dd4b86e3bf1b5f6c3ee52f5fe8213d6bc4d4df15e0", + installHash: "efdb08019c1e2d062dfae9dd4b86e3bf1b5f6c3ee52f5fe8213d6bc4d4df15e0", + integrity: + "sha512-AVl138zALtfaAPvADulE0PZThbYzCBS79nL4pOSL/6Sm/4AH5A21BD9VHt97OlCuzJuCpmeZtAtkinisF4Vb1g==", + }, +]; const ALLOWED_PATCHED_DEPENDENCIES = new Map([ ...LEGACY_PATCHED_DEPENDENCIES, - ...reviewedSecurityPatches.map(({ specifier, patchPath }) => [specifier, patchPath]), + ...[...reviewedSecurityPatches, ...REVIEWED_PRODUCT_PATCHES].map(({ specifier, patchPath }) => [ + specifier, + patchPath, + ]), ]); const REVIEWED_LOCK_PATCHES = new Map( - reviewedSecurityPatches.map(({ specifier, installHash }) => [specifier, installHash]), + [...reviewedSecurityPatches, ...REVIEWED_PRODUCT_PATCHES].map(({ specifier, installHash }) => [ + specifier, + installHash, + ]), ); const ALLOWED_PATCH_FILES = new Set([ @@ -125,6 +144,40 @@ function collectPatchFileViolations(cwd, violations) { } } +function verifyReviewedProductPatches(cwd) { + const workspace = readYamlFile(cwd, "pnpm-workspace.yaml"); + const lockfile = readYamlFile(cwd, "pnpm-lock.yaml"); + for (const artifact of REVIEWED_PRODUCT_PATCHES) { + const { specifier, patchPath, installHash } = artifact; + const patchFile = path.join(cwd, patchPath); + const snapshots = Object.keys(lockfile.snapshots ?? {}).filter( + (key) => key === specifier || key.startsWith(`${specifier}(`), + ); + const registered = workspace.patchedDependencies?.[specifier]; + const locked = lockfile.patchedDependencies?.[specifier]; + if (!registered && !locked && !fs.existsSync(patchFile) && snapshots.length === 0) { + continue; + } + if ( + registered !== patchPath || + locked !== installHash || + lockfile.packages?.[specifier]?.resolution?.integrity !== artifact.integrity || + snapshots.length === 0 || + snapshots.some((key) => { + const hashes = [...key.matchAll(/\(patch_hash=([^)]*)\)/gu)]; + return hashes.length !== 1 || hashes[0][1] !== installHash; + }) + ) { + throw new Error( + `Reviewed product patch registration or installation identity changed: ${specifier}`, + ); + } + if (createHash("sha256").update(fs.readFileSync(patchFile)).digest("hex") !== artifact.sha256) { + throw new Error(`Reviewed product patch bytes changed: ${specifier}`); + } + } +} + /** * Collects disallowed package patch declarations and patch files. */ @@ -162,8 +215,9 @@ export async function main() { ) .map(({ specifier }) => specifier), }); + verifyReviewedProductPatches(process.cwd()); process.stdout.write( - `PASS package patch guard: only approved pnpm patches; ${ALLOWED_PATCHED_DEPENDENCIES.size} registrations allowed with security artifact verification.\n`, + `PASS package patch guard: only approved pnpm patches; ${ALLOWED_PATCHED_DEPENDENCIES.size} registrations allowed with security artifact verification and product artifact verification.\n`, ); return; } diff --git a/test/scripts/check-package-patches.test.ts b/test/scripts/check-package-patches.test.ts index 5b4dba3cce..21a0a05075 100644 --- a/test/scripts/check-package-patches.test.ts +++ b/test/scripts/check-package-patches.test.ts @@ -1,5 +1,6 @@ // Check Package Patches tests cover check package patches script behavior. import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { copyFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -12,6 +13,9 @@ import { cleanupTempDirs, makeTempRepoRoot, writeJsonFile } from "../helpers/tem const tempDirs: string[] = []; const repoRoot = path.resolve(import.meta.dirname, "../.."); +const productPatchPath = "patches/pdfjs-dist@6.4.299.patch"; +const pdfjsIntegrity = + "sha512-AVl138zALtfaAPvADulE0PZThbYzCBS79nL4pOSL/6Sm/4AH5A21BD9VHt97OlCuzJuCpmeZtAtkinisF4Vb1g=="; const nestedGitEnvKeys = [ "GIT_ALTERNATE_OBJECT_DIRECTORIES", @@ -57,7 +61,11 @@ function makeSecurityRepo() { for (const name of ["pnpm-workspace.yaml", "pnpm-lock.yaml"]) { copyFileSync(path.join(repoRoot, name), path.join(dir, name)); } - for (const { patchPath } of getReviewedSecurityPatchRegistrations()) { + const patchPaths = [ + ...getReviewedSecurityPatchRegistrations().map(({ patchPath }) => patchPath), + productPatchPath, + ]; + for (const patchPath of patchPaths) { mkdirSync(path.dirname(path.join(dir, patchPath)), { recursive: true }); copyFileSync(path.join(repoRoot, patchPath), path.join(dir, patchPath)); } @@ -65,6 +73,38 @@ function makeSecurityRepo() { return dir; } +function makeProductRepo() { + const dir = makeRepo(); + const patch = readFileSync(path.join(repoRoot, productPatchPath)); + const hash = createHash("sha256").update(patch).digest("hex"); + mkdirSync(path.join(dir, "patches"), { recursive: true }); + writeFileSync(path.join(dir, productPatchPath), patch); + writeFileSync( + path.join(dir, "pnpm-workspace.yaml"), + `packages:\n - .\npatchedDependencies:\n pdfjs-dist@6.4.299: ${productPatchPath}\n`, + ); + writeFileSync( + path.join(dir, "pnpm-lock.yaml"), + `lockfileVersion: '9.0' +patchedDependencies: + pdfjs-dist@6.4.299: ${hash} +importers: + ui: + dependencies: + pdfjs-dist: + specifier: 6.4.299 + version: 6.4.299(patch_hash=${hash}) +packages: + pdfjs-dist@6.4.299: + resolution: {integrity: ${pdfjsIntegrity}} +snapshots: + pdfjs-dist@6.4.299(patch_hash=${hash}): {} +`, + ); + git(dir, ["add", "pnpm-workspace.yaml", "pnpm-lock.yaml", "patches"]); + return { dir, hash }; +} + function runPatchGuard(cwd: string) { return spawnSync(process.execPath, [path.join(repoRoot, "scripts/check-package-patches.mjs")], { cwd, @@ -78,6 +118,78 @@ afterEach(() => { }); describe("check-package-patches", () => { + it("verifies the approved PDF.js product patch without a security advisory registration", () => { + const { dir } = makeProductRepo(); + const securitySpecifiers = getReviewedSecurityPatchRegistrations().map( + ({ specifier }) => specifier, + ); + expect(securitySpecifiers).not.toContain("pdfjs-dist@6.4.299"); + expect(collectPackagePatchViolations(dir)).toEqual([]); + const result = runPatchGuard(dir); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("product artifact verification"); + }); + + it.each([ + "changed product bytes", + "missing product patch", + "wrong product version", + "wrong product path", + "wrong product lock hash", + "wrong product archive integrity", + "unpatched product snapshot", + "duplicate product patch hash", + "orphaned product patch", + ])("rejects %s at the actual CLI boundary", (change) => { + const { dir, hash } = makeProductRepo(); + const patchFile = path.join(dir, productPatchPath); + const workspacePath = path.join(dir, "pnpm-workspace.yaml"); + const lockPath = path.join(dir, "pnpm-lock.yaml"); + if (change === "changed product bytes") { + writeFileSync(patchFile, "unreviewed bytes\n"); + } else if (change === "missing product patch") { + rmSync(patchFile); + } else if (change === "wrong product version") { + for (const file of [workspacePath, lockPath]) { + writeFileSync(file, readFileSync(file, "utf8").replaceAll("6.4.299", "6.4.300")); + } + } else if (change === "wrong product path") { + writeFileSync( + workspacePath, + readFileSync(workspacePath, "utf8").replace(productPatchPath, "patches/other.patch"), + ); + } else if (change === "wrong product lock hash") { + writeFileSync(lockPath, readFileSync(lockPath, "utf8").replaceAll(hash, "0")); + } else if (change === "wrong product archive integrity") { + writeFileSync( + lockPath, + readFileSync(lockPath, "utf8").replace(pdfjsIntegrity, "sha512-unreviewed"), + ); + } else if (change === "unpatched product snapshot") { + writeFileSync( + lockPath, + readFileSync(lockPath, "utf8").replaceAll(`(patch_hash=${hash})`, ""), + ); + } else if (change === "duplicate product patch hash") { + writeFileSync( + lockPath, + readFileSync(lockPath, "utf8").replaceAll( + `(patch_hash=${hash})`, + `(patch_hash=${hash})(patch_hash=${hash})`, + ), + ); + } else if (change === "orphaned product patch") { + writeFileSync(workspacePath, "packages:\n - .\n"); + writeFileSync(lockPath, "lockfileVersion: '9.0'\nimporters:\n .:\npackages:\nsnapshots:\n"); + } + const result = runPatchGuard(dir); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stderr).not.toEqual(""); + expect(result.stderr).toContain(change === "missing product patch" ? "ENOENT" : "pdfjs-dist@"); + }); + it("admits the approved security artifacts only after actual CLI verification", () => { const dir = makeSecurityRepo(); expect(collectPackagePatchViolations(dir)).toEqual([]); diff --git a/test/scripts/control-ui-performance.test.ts b/test/scripts/control-ui-performance.test.ts index bc9387a5a1..390550669a 100644 --- a/test/scripts/control-ui-performance.test.ts +++ b/test/scripts/control-ui-performance.test.ts @@ -23,6 +23,7 @@ function createDistFixture() { sizes: { rawBytes: number; gzipBytes: number; brotliBytes: number }, ) => { const assetPath = path.join(assetsDir, file); + fs.mkdirSync(path.dirname(assetPath), { recursive: true }); fs.writeFileSync(assetPath, Buffer.alloc(sizes.rawBytes)); fs.writeFileSync(`${assetPath}.gz`, Buffer.alloc(sizes.gzipBytes)); fs.writeFileSync(`${assetPath}.br`, Buffer.alloc(sizes.brotliBytes)); @@ -42,6 +43,7 @@ function createMetrics(startupJsGzipBytes: number) { js: { requests: 1, rawBytes: 2_000, gzipBytes: startupJsGzipBytes, brotliBytes: 900 }, css: { requests: 1, rawBytes: 50, gzipBytes: 15, brotliBytes: 12 }, }, + workers: { js: { requests: 0, rawBytes: 0, gzipBytes: 0, brotliBytes: 0 } }, largest: { js: { file: "assets/index-a.js", @@ -67,6 +69,7 @@ const looseBudgets = { startupJsGzipBytes: 100_000, startupCssGzipBytes: 100_000, largestJsGzipBytes: 100_000, + largestWorkerJsGzipBytes: 100_000, largestCssGzipBytes: 100_000, }; @@ -122,7 +125,101 @@ describe("Control UI performance budgets", () => { expect(metrics.total.js).toMatchObject({ requests: 3, rawBytes: 380, gzipBytes: 135 }); expect(metrics.largest.js.file).toBe("assets/lazy-d.js"); expect(metrics.largest.css.file).toBe("assets/index-c.css"); + expect(metrics.workers).toEqual({ + js: { requests: 0, rawBytes: 0, gzipBytes: 0, brotliBytes: 0 }, + largestJs: undefined, + }); expect(formatControlUiPerformanceReport(metrics)).toContain("startup CSS: 1 request"); + expect(formatControlUiPerformanceReport(metrics)).toContain("largest worker JS: none"); + }); + + it.each([359, 361])( + "enforces the approved worker ceiling for a %i KiB nested worker", + (workerKiB) => { + const { distDir, writeAsset } = createDistFixture(); + fs.writeFileSync( + path.join(distDir, "index.html"), + '', + ); + writeAsset("index.js", { rawBytes: 100, gzipBytes: 40, brotliBytes: 30 }); + writeAsset("app.css", { rawBytes: 50, gzipBytes: 15, brotliBytes: 12 }); + writeAsset("workers/parser.js", { + rawBytes: 1_000_000, + gzipBytes: workerKiB * 1024, + brotliBytes: 200_000, + }); + writeAsset("workers/chunks/decoder.js", { rawBytes: 300, gzipBytes: 80, brotliBytes: 70 }); + const metrics = collectControlUiPerformanceMetrics(distDir); + expect(metrics.largest.js.file).toBe("assets/index.js"); + expect(metrics.workers.js).toMatchObject({ requests: 2, gzipBytes: workerKiB * 1024 + 80 }); + expect(metrics.workers.largestJs?.file).toBe("assets/workers/parser.js"); + expect(metrics.total.js).toMatchObject({ requests: 3, gzipBytes: workerKiB * 1024 + 120 }); + expect(metrics.startup.js).toMatchObject({ requests: 1, gzipBytes: 40 }); + expect(evaluateControlUiPerformanceBudgets(metrics).map((entry) => entry.metric)).toEqual( + workerKiB > 360 ? ["largest worker JS gzip"] : [], + ); + expect(formatControlUiPerformanceReport(metrics)).toContain( + `largest worker JS: assets/workers/parser.js, ${workerKiB}.0 KiB gzip (limit: 360.0 KiB)`, + ); + }, + ); + + it.each(["lazy/large.js", "lazy/large.mjs", "workers-adjacent/large.js"])( + "keeps the ordinary ceiling for nested main-thread %s", + (filename) => { + const { distDir, writeAsset } = createDistFixture(); + fs.writeFileSync( + path.join(distDir, "index.html"), + '', + ); + writeAsset("index.js", { rawBytes: 100, gzipBytes: 40, brotliBytes: 30 }); + writeAsset("theme/app.css", { rawBytes: 50, gzipBytes: 15, brotliBytes: 12 }); + writeAsset(filename, { rawBytes: 1_000_000, gzipBytes: 216 * 1024, brotliBytes: 100_000 }); + const metrics = collectControlUiPerformanceMetrics(distDir); + expect(metrics.largest.js.file).toBe(`assets/${filename}`); + expect(metrics.largest.css.file).toBe("assets/theme/app.css"); + expect(evaluateControlUiPerformanceBudgets(metrics).map((entry) => entry.metric)).toEqual([ + "largest JS gzip", + ]); + expect(metrics.workers.js.requests).toBe(0); + }, + ); + + it.each([ + '', + '', + '', + ])("rejects worker assets referenced by startup HTML: %s", (html) => { + const { distDir, writeAsset } = createDistFixture(); + fs.writeFileSync(path.join(distDir, "index.html"), html); + writeAsset("index.js", { rawBytes: 100, gzipBytes: 40, brotliBytes: 30 }); + writeAsset("app.css", { rawBytes: 50, gzipBytes: 15, brotliBytes: 12 }); + writeAsset("workers/parser.js", { rawBytes: 100, gzipBytes: 40, brotliBytes: 30 }); + expect(() => collectControlUiPerformanceMetrics(distDir)).toThrow( + "worker asset must not appear in startup HTML", + ); + }); + + it.each([ + ["workers/parser.js", ".gz"], + ["workers/parser.js", ".br"], + ["nested/main.js", ".gz"], + ["nested/main.js", ".br"], + ["nested/theme.css", ".gz"], + ["nested/theme.css", ".br"], + ])("requires the %s%s compressed sidecar", (filename, suffix) => { + const { distDir, writeAsset } = createDistFixture(); + fs.writeFileSync( + path.join(distDir, "index.html"), + '', + ); + writeAsset("index.js", { rawBytes: 100, gzipBytes: 40, brotliBytes: 30 }); + writeAsset("app.css", { rawBytes: 50, gzipBytes: 15, brotliBytes: 12 }); + writeAsset(filename, { rawBytes: 100, gzipBytes: 30, brotliBytes: 20 }); + fs.rmSync(path.join(distDir, "assets", `${filename}${suffix}`)); + expect(() => collectControlUiPerformanceMetrics(distDir)).toThrow( + `missing ${filename}${suffix}`, + ); }); it("returns actionable violations and includes them in the report", () => { @@ -141,6 +238,7 @@ describe("Control UI performance budgets", () => { startupJsGzipBytes: 30, startupCssGzipBytes: 20, largestJsGzipBytes: 35, + largestWorkerJsGzipBytes: 100_000, largestCssGzipBytes: 20, }; @@ -164,6 +262,7 @@ describe("Control UI performance budgets", () => { js: { requests: 1, rawBytes: 100, gzipBytes: 43_009, brotliBytes: 30 }, css: { requests: 1, rawBytes: 50, gzipBytes: 15, brotliBytes: 12 }, }, + workers: { js: { requests: 0, rawBytes: 0, gzipBytes: 0, brotliBytes: 0 } }, largest: { js: { file: "assets/index-a.js", @@ -187,6 +286,7 @@ describe("Control UI performance budgets", () => { startupJsGzipBytes: 43_008, startupCssGzipBytes: 20, largestJsGzipBytes: 43_008, + largestWorkerJsGzipBytes: 100_000, largestCssGzipBytes: 20, }; diff --git a/test/scripts/control-ui-precompressed-assets.test.ts b/test/scripts/control-ui-precompressed-assets.test.ts new file mode 100644 index 0000000000..383df649d3 --- /dev/null +++ b/test/scripts/control-ui-precompressed-assets.test.ts @@ -0,0 +1,61 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { brotliCompressSync, gzipSync } from "node:zlib"; +import { afterEach, describe, expect, it } from "vitest"; + +const temporaryRoots: string[] = []; + +function createAssetFixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "control-ui-sidecars-")); + temporaryRoots.push(root); + fs.mkdirSync(path.join(root, ".git")); + fs.mkdirSync(path.join(root, "scripts/lib"), { recursive: true }); + for (const file of ["check-control-ui-precompressed-assets.mjs", "lib/repo-root.mjs"]) { + fs.copyFileSync(path.resolve("scripts", file), path.join(root, "scripts", file)); + } + const assets = path.join(root, "dist/control-ui/assets"); + fs.mkdirSync(path.join(assets, "workers"), { recursive: true }); + fs.mkdirSync(path.join(assets, "pdfjs/fonts"), { recursive: true }); + for (const file of ["index.js", "workers/document.js", "pdfjs/fonts/LICENSE.txt"]) { + const source = Buffer.from(`Final emitted bytes for ${file}`); + fs.writeFileSync(path.join(assets, file), source); + fs.writeFileSync(path.join(assets, `${file}.gz`), gzipSync(source)); + fs.writeFileSync(path.join(assets, `${file}.br`), brotliCompressSync(source)); + } + const run = () => + spawnSync( + process.execPath, + [path.join(root, "scripts/check-control-ui-precompressed-assets.mjs")], + { cwd: root, encoding: "utf8" }, + ); + return { assets, run }; +} + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +describe("finalized Control UI sidecar verification", () => { + it("verifies both encodings for ordinary, worker, and nested PDF resources", () => { + const fixture = createAssetFixture(); + const result = fixture.run(); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("verified 6 finalized Control UI sidecars"); + }); + + it.each(["workers/document.js", "pdfjs/fonts/LICENSE.txt"])( + "rejects a stale nested sidecar for %s through the actual CLI", + (file) => { + const fixture = createAssetFixture(); + fs.writeFileSync(path.join(fixture.assets, file), "Changed after compression"); + const result = fixture.run(); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("decoded bytes differ"); + expect(result.stderr).toContain(path.normalize(file)); + }, + ); +}); diff --git a/test/vitest/vitest.ui-isolated.config.ts b/test/vitest/vitest.ui-isolated.config.ts index 3bfdd44a84..21600a4202 100644 --- a/test/vitest/vitest.ui-isolated.config.ts +++ b/test/vitest/vitest.ui-isolated.config.ts @@ -1,6 +1,7 @@ // Vitest ui-isolated config preserves the dedicated UI project partition. // Both UI projects use native per-file DOM and module isolation. import { controlUiLocaleModulesPlugin } from "../../ui/config/control-ui-locales.ts"; +import { controlUiPdfAssetsPlugin } from "../../ui/config/control-ui-pdf-assets.ts"; import { nativeNodeTestBuiltinsPlugin } from "./vitest.native-node-builtins.ts"; import { createScopedVitestConfig } from "./vitest.scoped-config.ts"; import { jsdomOptimizedDeps } from "./vitest.shared.config.ts"; @@ -24,6 +25,7 @@ export function createUiIsolatedVitestConfig(env?: Record (left.name < right.name ? -1 : left.name > right.name ? 1 : 0)) + .flatMap((entry) => { + const filename = path.join(directory, entry.name); + if (entry.isDirectory()) { + return collectFiles(filename); + } + if (!entry.isFile()) { + throw new Error(`PDF.js assets must be regular files: ${filename}`); + } + return [filename]; + }); +} + +/** Keep PDF.js resources local and version-matched to the UI-owned library. */ +export function controlUiPdfAssetsPlugin(): Plugin { + let building = false; + return { + name: "control-ui-pdf-assets", + configResolved(config) { + building = config.command === "build"; + }, + resolveId(source) { + return source === VIRTUAL_MODULE ? RESOLVED_MODULE : null; + }, + load(id) { + if (id !== RESOLVED_MODULE) { + return null; + } + const packageFile = require.resolve("pdfjs-dist/package.json"); + const packageRoot = path.dirname(packageFile); + const metadata: unknown = JSON.parse(readFileSync(packageFile, "utf8")); + if ( + !metadata || + typeof metadata !== "object" || + !("version" in metadata) || + typeof metadata.version !== "string" || + !/^\d+\.\d+\.\d+$/u.test(metadata.version) + ) { + throw new Error("PDF.js package version is missing or invalid"); + } + const version = metadata.version; + this.addWatchFile(packageFile); + if (building) { + const licenseFile = path.join(packageRoot, "LICENSE"); + this.addWatchFile(licenseFile); + this.emitFile({ + type: "asset", + fileName: `assets/pdfjs-${version}/LICENSE`, + source: readFileSync(licenseFile), + }); + } + const properties = Object.entries(DIRECTORIES).map(([property, directory]) => { + const directoryRoot = path.join(packageRoot, directory); + if (!building) { + const url = `/@fs/${normalizePath(directoryRoot)}/`; + return `${JSON.stringify(property)}: ${JSON.stringify(url)}`; + } + let marker: string | undefined; + for (const filename of collectFiles(directoryRoot)) { + this.addWatchFile(filename); + const relative = normalizePath(path.relative(directoryRoot, filename)); + const reference = this.emitFile({ + type: "asset", + fileName: `assets/pdfjs-${version}/${directory}/${relative}`, + source: readFileSync(filename), + }); + if (!marker && path.dirname(filename) === directoryRoot) { + marker = reference; + } + } + if (!marker) { + throw new Error(`PDF.js asset directory has no root files: ${directory}`); + } + // Let Vite resolve an emitted file URL relative to its owning chunk; + // deriving the directory from that URL preserves mounted app bases. + const url = `new URL("./", import.meta.ROLLUP_FILE_URL_${marker}).href`; + return `${JSON.stringify(property)}: ${url}`; + }); + return `export const pdfAssetUrls = {\n${properties.join(",\n")}\n};\n`; + }, + }; +} diff --git a/ui/package.json b/ui/package.json index 0d8ff7208a..760c01831f 100644 --- a/ui/package.json +++ b/ui/package.json @@ -40,6 +40,7 @@ "markdown-it": "14.3.3", "markdown-it-task-lists": "2.1.1", "marked": "18.0.7", + "pdfjs-dist": "6.4.299", "remend": "1.3.0" }, "devDependencies": { diff --git a/ui/src/app/pdf-assets.node.test.ts b/ui/src/app/pdf-assets.node.test.ts new file mode 100644 index 0000000000..d0dd9cb630 --- /dev/null +++ b/ui/src/app/pdf-assets.node.test.ts @@ -0,0 +1,106 @@ +// @vitest-environment node +import { execFileSync } from "node:child_process"; +import { + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { build, normalizePath } from "vite"; +import { describe, expect, it } from "vitest"; +import { controlUiPdfAssetsPlugin } from "../../config/control-ui-pdf-assets.ts"; + +const require = createRequire(import.meta.url); + +describe("local PDF.js assets", () => { + it("emits all resources and resolves directory URLs in a real relative build", async () => { + const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "control-ui-pdf-assets-"))); + try { + const entry = path.join(root, "entry.mjs"); + const outDir = path.join(root, "dist"); + writeFileSync(entry, 'export { pdfAssetUrls } from "virtual:openclaw-pdf-assets";\n'); + const result = await build({ + root, + configFile: false, + base: "./", + logLevel: "silent", + plugins: [controlUiPdfAssetsPlugin()], + build: { + outDir, + minify: false, + rolldownOptions: { + input: entry, + preserveEntrySignatures: "strict", + output: { entryFileNames: "assets/preview.mjs" }, + }, + }, + }); + const bundle = Array.isArray(result) ? result[0] : result; + if (!bundle || !("output" in bundle)) { + throw new Error("Expected a completed PDF asset build"); + } + const packageFile = require.resolve("pdfjs-dist/package.json"); + const packageRoot = path.dirname(packageFile); + const metadata: unknown = JSON.parse(readFileSync(packageFile, "utf8")); + if (!metadata || typeof metadata !== "object" || !("version" in metadata)) { + throw new Error("Expected PDF.js package metadata"); + } + expect(metadata.version).toBe("6.4.299"); + const prefix = "assets/pdfjs-6.4.299"; + expect(readFileSync(path.join(outDir, prefix, "LICENSE"))).toEqual( + readFileSync(path.join(packageRoot, "LICENSE")), + ); + const expectedFiles = [`${prefix}/LICENSE`]; + const expectedUrls: Record = {}; + for (const [property, directory] of Object.entries({ + cMapUrl: "cmaps", + standardFontDataUrl: "standard_fonts", + wasmUrl: "wasm", + iccUrl: "iccs", + })) { + const sourceRoot = path.join(packageRoot, directory); + const outputRoot = path.join(outDir, prefix, directory); + expectedUrls[property] = pathToFileURL(`${outputRoot}${path.sep}`).href; + const files = readdirSync(sourceRoot, { recursive: true, withFileTypes: true }); + expect(files.some((file) => file.name.startsWith("LICENSE"))).toBe(true); + for (const file of files) { + if (!file.isFile()) { + continue; + } + const source = path.join(file.parentPath, file.name); + const relative = path.relative(sourceRoot, source); + const output = path.join(outputRoot, relative); + expect(readFileSync(output)).toEqual(readFileSync(source)); + expectedFiles.push(`${prefix}/${directory}/${normalizePath(relative)}`); + } + } + expect( + bundle.output + .filter((output) => output.type === "asset") + .map((output) => output.fileName) + .toSorted(), + ).toEqual(expectedFiles.toSorted()); + // A native import preserves import.meta.url instead of Vitest rewriting it. + const exported = execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `const {pdfAssetUrls} = await import(process.argv[1]); +process.stdout.write(JSON.stringify(pdfAssetUrls));`, + pathToFileURL(path.join(outDir, "assets/preview.mjs")).href, + ], + { encoding: "utf8" }, + ); + expect(JSON.parse(exported)).toEqual(expectedUrls); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/ui/src/app/vite-config.node.test.ts b/ui/src/app/vite-config.node.test.ts index 6de3dcfdb3..dcdf73903c 100644 --- a/ui/src/app/vite-config.node.test.ts +++ b/ui/src/app/vite-config.node.test.ts @@ -92,13 +92,17 @@ describe("Control UI Vite config", () => { } }); - it("emits Brotli and gzip variants only for bundled compressible assets", () => { + it.each([ + "assets/app-AbCd1234.js", + "assets/workers/document-AbCd1234.js", + "assets/lazy-AbCd1234.mjs", + ])("emits byte-exact Brotli and gzip variants for %s", (fileName) => { const source = "console.log('precompressed');\n".repeat(200); - const variants = createControlUiPrecompressedAssetVariants("assets/app-AbCd1234.js", source); + const variants = createControlUiPrecompressedAssetVariants(fileName, source); expect(variants.map((variant) => variant.fileName)).toEqual([ - "assets/app-AbCd1234.js.br", - "assets/app-AbCd1234.js.gz", + `${fileName}.br`, + `${fileName}.gz`, ]); expect(brotliDecompressSync(variants[0]?.source ?? Buffer.alloc(0)).toString()).toBe(source); expect(gunzipSync(variants[1]?.source ?? Buffer.alloc(0)).toString()).toBe(source); @@ -107,6 +111,43 @@ describe("Control UI Vite config", () => { expect(createControlUiPrecompressedAssetVariants("assets/app.js.map", source)).toEqual([]); }); + it("emits actual worker code into the reserved off-main-thread asset directory", async () => { + const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "control-ui-worker-build-"))); + try { + writeFileSync( + path.join(root, "index.html"), + '', + ); + writeFileSync( + path.join(root, "entry.js"), + 'new Worker(new URL("./document-worker.js", import.meta.url), { type: "module" });', + ); + writeFileSync(path.join(root, "document-worker.js"), 'self.postMessage("ready");'); + const result = await build({ + root, + configFile: false, + logLevel: "silent", + worker: controlUiViteConfig().worker, + build: { write: false }, + }); + const bundle = Array.isArray(result) ? result[0] : result; + if (!bundle || !("output" in bundle)) { + throw new Error("Expected a completed worker build"); + } + const worker = bundle.output.find((output) => + /^assets\/workers\/document-worker-[\w-]+\.js$/u.test(output.fileName), + ); + if (!worker || worker.type !== "asset") { + throw new Error("Worker code did not use the reserved build output directory"); + } + expect(String(worker.source)).toContain("postMessage"); + const entry = bundle.output.find((output) => output.type === "chunk" && output.isEntry); + expect(entry && entry.type === "chunk" ? entry.code : "").toContain("assets/workers/"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it("embeds one canonical artifact identity from explicit build inputs", () => { const readGitCommit = vi.fn(() => "f".repeat(40)); const readGitCommitTimestamp = vi.fn(() => "2026-07-10T10:11:12.000Z"); diff --git a/ui/src/components/document-viewer-contents.styles.ts b/ui/src/components/document-viewer-contents.styles.ts new file mode 100644 index 0000000000..abcb1674d4 --- /dev/null +++ b/ui/src/components/document-viewer-contents.styles.ts @@ -0,0 +1,102 @@ +import { css } from "lit"; + +export const documentContentsStyles = css` + :host { + display: block; + width: 156px; + min-width: 0; + height: 100%; + color: var(--text, #222); + } + nav { + display: flex; + flex-direction: column; + height: 100%; + background: var(--bg, #fff); + } + .window-controls { + display: flex; + align-items: center; + justify-content: space-between; + gap: 4px; + padding: 8px; + font-size: 11px; + } + button { + color: inherit; + background: var(--bg, #fff); + border: 1px solid var(--border, #ccc); + border-radius: 5px; + cursor: pointer; + } + button:disabled { + opacity: 0.4; + cursor: default; + } + button:focus-visible { + outline: 2px solid var(--accent, #159b93); + outline-offset: 2px; + } + .window-controls button { + width: 28px; + min-height: 30px; + } + .thumbnail-list { + display: flex; + flex-direction: column; + gap: 8px; + padding: 4px 8px 12px; + min-height: 0; + overflow: auto; + } + .thumbnail { + display: flex; + flex-direction: column; + align-items: center; + gap: 4px; + min-height: 64px; + padding: 6px; + font: inherit; + font-size: 12px; + } + .thumbnail[aria-current="page"] { + border: 2px solid var(--accent, #159b93); + background: var(--bg-hover, #eee); + } + canvas { + display: block; + max-width: 100%; + max-height: 144px; + background: #fff; + } + canvas[data-thumbnail-state="loading"] { + visibility: hidden; + height: 96px; + } + .thumbnail-error { + font-size: 11px; + } + @media (max-width: 600px) { + :host { + width: 100%; + height: 156px; + } + .window-controls { + padding: 4px 8px; + } + .thumbnail-list { + flex-direction: row; + padding: 4px 8px; + } + .thumbnail { + flex: 0 0 84px; + min-width: 0; + } + canvas { + max-height: 84px; + } + canvas[data-thumbnail-state="loading"] { + height: 68px; + } + } +`; diff --git a/ui/src/components/document-viewer-contents.ts b/ui/src/components/document-viewer-contents.ts new file mode 100644 index 0000000000..80c8a572b4 --- /dev/null +++ b/ui/src/components/document-viewer-contents.ts @@ -0,0 +1,247 @@ +import { html, nothing, type PropertyValues } from "lit"; +import { property, state } from "lit/decorators.js"; +import { repeat } from "lit/directives/repeat.js"; +import { t } from "../i18n/index.ts"; +import { OpenClawLitElement } from "../lit/openclaw-element.ts"; +import { documentContentsStyles } from "./document-viewer-contents.styles.ts"; +import type { + PDFDocumentProxy, + PDFPageProxy, + RenderTask, + renderPdfPage, +} from "./pdf-document.runtime.ts"; + +const CONTENTS_WINDOW_PAGES = 6; +const THUMBNAIL_MAX_DIMENSION = 144; +const THUMBNAIL_WINDOW_TIMEOUT_MS = 30_000; + +/** Only the open six-page window renders thumbnails, serially on the existing PDF worker. */ +class OpenClawDocumentContents extends OpenClawLitElement { + @property({ attribute: false }) pdf: PDFDocumentProxy | undefined; + @property({ attribute: false }) renderPage: typeof renderPdfPage | undefined; + @property({ type: Number }) currentPage = 1; + @property() pageUnit: "page" | "slide" = "page"; + @property({ type: Boolean }) disabled = false; + @state() private windowStart = 1; + @state() private completed = new Set(); + @state() private failed = new Set(); + private task: RenderTask | undefined; + private generation = 0; + private deadline: ReturnType | undefined; + private readonly canvases = new Set(); + + static override styles = documentContentsStyles; + + override disconnectedCallback(): void { + void this.retireWindow(); + super.disconnectedCallback(); + } + + override connectedCallback(): void { + super.connectedCallback(); + if (this.hasUpdated) { + void this.renderWindow(); + } + } + + protected override willUpdate(changed: PropertyValues): void { + if (changed.has("pdf")) { + this.windowStart = 1; + } + if ( + changed.has("currentPage") && + (this.currentPage < this.windowStart || + this.currentPage >= this.windowStart + CONTENTS_WINDOW_PAGES) + ) { + this.windowStart = + Math.floor((this.currentPage - 1) / CONTENTS_WINDOW_PAGES) * CONTENTS_WINDOW_PAGES + 1; + } + } + + protected override updated(changed: PropertyValues): void { + if ( + changed.has("pdf") || + changed.has("renderPage") || + changed.has("windowStart") || + (changed.has("currentPage") && + !this.completed.has(this.currentPage) && + !this.failed.has(this.currentPage)) + ) { + void this.renderWindow(); + } + } + + private async retireWindow(): Promise { + this.generation++; + clearTimeout(this.deadline); + this.deadline = undefined; + const task = this.task; + this.task = undefined; + task?.cancel(); + for (const canvas of this.canvases) { + canvas.width = canvas.height = 0; + } + this.canvases.clear(); + await task?.promise.catch(() => undefined); + } + + private pages(): number[] { + const count = Math.min(CONTENTS_WINDOW_PAGES, (this.pdf?.numPages ?? 0) - this.windowStart + 1); + return Array.from({ length: Math.max(0, count) }, (_, index) => this.windowStart + index); + } + + private async renderWindow(): Promise { + const retired = this.retireWindow(); + const generation = this.generation; + const pdf = this.pdf; + const renderPage = this.renderPage; + const pages = this.pages(); + const current = () => this.isConnected && generation === this.generation; + this.completed = new Set(); + this.failed = new Set(); + if (!pdf || !renderPage || !pages.length) { + return; + } + await retired; + await this.updateComplete; + if (!current()) { + return; + } + let expired = false; + this.deadline = setTimeout(() => { + if (current()) { + expired = true; + this.task?.cancel(); + this.failed = new Set(pages.filter((number) => !this.completed.has(number))); + } + }, THUMBNAIL_WINDOW_TIMEOUT_MS); + try { + for (const number of pages) { + const canvas = this.shadowRoot?.querySelector( + `canvas[data-page="${number}"]`, + ); + if (!current() || expired || !canvas) { + return; + } + let page: PDFPageProxy | undefined; + try { + page = await pdf.getPage(number); + if (!current() || expired) { + return; + } + const base = page.getViewport({ scale: 1 }); + const scale = THUMBNAIL_MAX_DIMENSION / Math.max(base.width, base.height); + if (!Number.isFinite(scale) || scale <= 0) { + throw new Error("Invalid thumbnail dimensions"); + } + const viewport = page.getViewport({ scale }); + this.canvases.add(canvas); + canvas.width = Math.max(1, Math.floor(viewport.width)); + canvas.height = Math.max(1, Math.floor(viewport.height)); + const task = renderPage(page, canvas, viewport); + this.task = task; + await task.promise; + if (!current() || expired) { + return; + } + this.task = undefined; + this.completed = new Set([...this.completed, number]); + } catch { + if (!current() || expired) { + return; + } + this.task = undefined; + canvas.width = canvas.height = 0; + this.failed = new Set([...this.failed, number]); + } finally { + // The main page owns its shared PDFPageProxy; other thumbnails release + // parsed operators after their bounded canvas has retained the pixels. + if (number !== this.currentPage) { + page?.cleanup(); + } + } + } + } finally { + if (current()) { + clearTimeout(this.deadline); + this.deadline = undefined; + } + } + } + + override render() { + const total = this.pdf?.numPages ?? 0; + return html``; + } +} + +if (!customElements.get("openclaw-document-contents")) { + customElements.define("openclaw-document-contents", OpenClawDocumentContents); +} + +declare global { + interface HTMLElementTagNameMap { + "openclaw-document-contents": OpenClawDocumentContents; + } +} diff --git a/ui/src/components/document-viewer-request.test.ts b/ui/src/components/document-viewer-request.test.ts new file mode 100644 index 0000000000..aa5f60be60 --- /dev/null +++ b/ui/src/components/document-viewer-request.test.ts @@ -0,0 +1,398 @@ +/* @vitest-environment jsdom */ +import { html, render } from "lit"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + DOCUMENT_CHANGE_CROP_MAX_BYTES, + DOCUMENT_CHANGE_INSTRUCTION_MAX, + DOCUMENT_CHANGE_TEXT_MAX, + DocumentViewerRequestController, + type DocumentChangeHandler, + type DocumentChangeResult, +} from "./document-viewer-request.ts"; +import type { DocumentSelectionTarget } from "./document-viewer-selection.ts"; + +const PNG = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jP3sAAAAASUVORK5CYII="; +const drawImage = vi.fn<(...args: unknown[]) => void>(); +const resizeObservers: TestResizeObserver[] = []; +const requestControllers: DocumentViewerRequestController[] = []; +let originalGetContext: PropertyDescriptor | undefined; +class TestResizeObserver { + disconnected = false; + constructor(readonly notify: () => void) { + resizeObservers.push(this); + } + observe() {} + disconnect(): void { + this.disconnected = true; + } +} + +function deferred() { + let resolve: (value: T) => void = () => {}; + const promise = new Promise((accept) => { + resolve = accept; + }); + return { promise, resolve }; +} + +function mountRequest(options: { stageBounds?: DOMRect; canvasBounds?: DOMRect } = {}) { + const container = document.createElement("div"); + document.body.append(container); + const canvas = document.createElement("canvas"); + canvas.width = 1_200; + canvas.height = 800; + let pageNumber = 1; + let targets: readonly DocumentSelectionTarget[] = [ + { region: { x: 0.1, y: 0.1, width: 0.7, height: 0.2 }, text: "A larger text run" }, + { region: { x: 0.2, y: 0.1, width: 0.2, height: 0.08 }, text: "Owned title" }, + ]; + const handler = vi + .fn() + .mockResolvedValue({ admitted: false, message: "Choose the current document." }); + const stageBounds = options.stageBounds ?? new DOMRect(80, 60, 640, 500); + const canvasBounds = options.canvasBounds ?? new DOMRect(100, 80, 600, 400); + let stage: HTMLDivElement | undefined; + const measuredForms = new WeakSet(); + vi.spyOn(canvas, "getBoundingClientRect").mockImplementation( + () => + new DOMRect( + canvasBounds.x - (stage?.scrollLeft ?? 0), + canvasBounds.y - (stage?.scrollTop ?? 0), + canvasBounds.width, + canvasBounds.height, + ), + ); + const refresh = () => { + render( + html`
+
+ ${canvas}${controller.renderOverlay()} +
+ ${controller.renderPopover()} +
+ ${controller.renderStatus()}`, + container, + ); + stage = container.querySelector(".stage") ?? undefined; + const surface = container.querySelector(".page-surface"); + if (!stage || !surface) { + throw new Error("Missing document stage"); + } + // jsdom does not lay out canvases or provide native pointer capture. These + // geometry/capture adapters let real DOM events exercise the controller; + // pixel correctness and actual layout belong to the Chromium scenario. + Object.defineProperties(stage, { + clientWidth: { configurable: true, value: stageBounds.width }, + clientHeight: { configurable: true, value: stageBounds.height }, + }); + vi.spyOn(stage, "getBoundingClientRect").mockReturnValue(stageBounds); + surface.setPointerCapture = vi.fn(); + const form = container.querySelector("form"); + if (form && !measuredForms.has(form)) { + measuredForms.add(form); + vi.spyOn(form, "getBoundingClientRect").mockImplementation( + () => + new DOMRect( + stageBounds.x + + Number.parseFloat(form.style.getPropertyValue("--request-popover-left") || "0") - + (stage?.scrollLeft ?? 0), + stageBounds.y + + Number.parseFloat(form.style.getPropertyValue("--request-popover-top") || "0") - + (stage?.scrollTop ?? 0), + Number.parseFloat(form.style.getPropertyValue("--request-popover-width") || "300"), + Math.min( + 160, + Number.parseFloat( + form.style.getPropertyValue("--request-popover-max-height") || "160", + ), + ), + ), + ); + } + }; + const controller = new DocumentViewerRequestController({ + requestUpdate: () => { + queueMicrotask(refresh); + }, + currentPage: () => + container.isConnected ? { number: pageNumber, canvas, targets } : undefined, + handler: () => handler, + }); + requestControllers.push(controller); + refresh(); + const pointer = (type: string, x: number, y: number) => { + const bounds = canvas.getBoundingClientRect(); + container.querySelector(".page-surface")?.dispatchEvent( + new PointerEvent(type, { + pointerId: 1, + button: 0, + clientX: bounds.left + bounds.width * x, + clientY: bounds.top + bounds.height * y, + bubbles: true, + }), + ); + }; + return { + container, + canvas, + controller, + handler, + pointer, + setTargets: (value: readonly DocumentSelectionTarget[]) => { + targets = value; + }, + select: async (x = 0.25, y = 0.13) => { + pointer("pointerdown", x, y); + pointer("pointerup", x, y); + if (controller.active) { + await vi.waitFor(() => expect(container.querySelector("form")).not.toBeNull()); + } + }, + instruction: async (value: string) => { + const input = container.querySelector("textarea"); + if (!input) { + throw new Error("Missing explicit request input"); + } + input.value = value; + input.dispatchEvent(new Event("input", { bubbles: true })); + await Promise.resolve(); + return input; + }, + changePage: async (number: number) => { + pageNumber = number; + controller.changePage(number); + await Promise.resolve(); + }, + }; +} + +beforeEach(() => { + resizeObservers.length = 0; + requestControllers.length = 0; + vi.stubGlobal("ResizeObserver", TestResizeObserver); + // The controller's Canvas2D boundary only needs drawImage. This narrow adapter + // controls encoding success/failure; it does not claim to validate PDF pixels. + originalGetContext = Object.getOwnPropertyDescriptor(HTMLCanvasElement.prototype, "getContext"); + Object.defineProperty(HTMLCanvasElement.prototype, "getContext", { + configurable: true, + value(kind: string): Pick | null { + return kind === "2d" ? { drawImage } : null; + }, + }); + vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValue(PNG); + drawImage.mockClear(); +}); + +afterEach(() => { + requestControllers.forEach((controller) => controller.reset()); + document.body.replaceChildren(); + vi.restoreAllMocks(); + if (originalGetContext) { + Object.defineProperty(HTMLCanvasElement.prototype, "getContext", originalGetContext); + } else { + Reflect.deleteProperty(HTMLCanvasElement.prototype, "getContext"); + } + vi.unstubAllGlobals(); +}); + +function submit(container: HTMLElement): void { + container.querySelector("form")?.dispatchEvent(new Event("submit", { cancelable: true })); +} + +describe("inline document selection requests", () => { + it("ignores ordinary clicks and only opens a popover after an explicit mode selection", async () => { + const request = mountRequest(); + await request.select(); + expect(request.container.querySelector("form")).toBeNull(); + request.controller.begin(); + await Promise.resolve(); + expect(request.container.querySelector("form")).toBeNull(); + await request.select(); + expect(request.container.querySelector(".selected-text")?.textContent).toContain("Owned title"); + expect(request.container.querySelector(".request-popover")?.closest(".stage")).not.toBeNull(); + expect(request.handler).not.toHaveBeenCalled(); + }); + + it("uses the smallest hit text box and retains a bounded refused request", async () => { + const request = mountRequest(); + request.setTargets([ + { + region: { x: 0.2, y: 0.1, width: 0.2, height: 0.08 }, + text: "A".repeat(DOCUMENT_CHANGE_TEXT_MAX + 100), + }, + ]); + request.controller.begin(); + await request.select(); + const input = await request.instruction("I".repeat(DOCUMENT_CHANGE_INSTRUCTION_MAX + 100)); + submit(request.container); + await vi.waitFor(() => + expect(request.container.querySelector('[role="alert"]')?.textContent).toContain( + "Choose the current document.", + ), + ); + expect(input.value).toHaveLength(DOCUMENT_CHANGE_INSTRUCTION_MAX); + expect(request.handler.mock.calls[0]?.[0]).toMatchObject({ + pageNumber: 1, + region: { x: 0.2, y: 0.1, width: 0.2, height: 0.08 }, + cropDataUrl: PNG, + }); + expect(request.handler.mock.calls[0]?.[0].selectedText).toHaveLength(DOCUMENT_CHANGE_TEXT_MAX); + expect(request.handler.mock.calls[0]?.[0].instruction).toHaveLength( + DOCUMENT_CHANGE_INSTRUCTION_MAX, + ); + expect(request.container.querySelector(".selected-text")).not.toBeNull(); + expect(request.container.querySelector('button[type="submit"]')?.hasAttribute("disabled")).toBe( + false, + ); + }); + + it("uses a clamped point crop on blank content and preserves actual drag regions", async () => { + const request = mountRequest(); + request.controller.begin(); + await request.select(0.99, 0.99); + await request.instruction("Change this corner."); + submit(request.container); + await vi.waitFor(() => expect(request.handler).toHaveBeenCalledTimes(1)); + expect(request.handler.mock.calls[0]?.[0]).toMatchObject({ + region: { width: 0.35, height: 0.18 }, + }); + expect(request.handler.mock.calls[0]?.[0].region?.x).toBeCloseTo(0.65); + expect(request.handler.mock.calls[0]?.[0].region?.y).toBeCloseTo(0.82); + expect(request.handler.mock.calls[0]?.[0].selectedText).toBeUndefined(); + await vi.waitFor(() => expect(request.controller.busy).toBe(false)); + request.pointer("pointerdown", 0.15, 0.15); + request.pointer("pointermove", 0.65, 0.55); + await Promise.resolve(); + expect(request.container.querySelector("form")).toBeNull(); + request.pointer("pointerup", 0.65, 0.55); + await vi.waitFor(() => expect(request.container.querySelector("form")).not.toBeNull()); + submit(request.container); + await vi.waitFor(() => expect(request.handler).toHaveBeenCalledTimes(2)); + expect(request.handler.mock.calls[1]?.[0].region?.width).toBeCloseTo(0.5); + expect(request.handler.mock.calls[1]?.[0].region?.height).toBeCloseTo(0.4); + expect(request.handler.mock.calls[1]?.[0].selectedText).toBeUndefined(); + }); + + it("dismisses selection without ending the mode and releases popover layout listeners", async () => { + const request = mountRequest(); + request.controller.begin(); + await request.select(); + await request.instruction("Discard this draft."); + request.container.querySelector('[aria-label="Cancel request"]')?.click(); + await Promise.resolve(); + expect(request.container.querySelector("form")).toBeNull(); + expect(request.controller.active).toBe(true); + expect(resizeObservers.every((observer) => observer.disconnected)).toBe(true); + await request.select(); + expect(request.container.querySelector("textarea")?.value).toBe(""); + request.controller.reset(); + await Promise.resolve(); + expect(request.controller.active).toBe(false); + expect(request.container.querySelector("form")).toBeNull(); + expect(request.handler).not.toHaveBeenCalled(); + }); + + it("hides stale page selection while preserving instructions for a new selection", async () => { + const request = mountRequest(); + request.controller.begin(); + await request.select(); + await request.instruction("Keep these instructions."); + await request.changePage(2); + expect(request.container.querySelector("form")).toBeNull(); + await request.select(); + expect(request.container.querySelector("textarea")?.value).toBe("Keep these instructions."); + expect(request.handler).not.toHaveBeenCalled(); + }); + + it("clears an admitted request exactly once and ignores a retired pending completion", async () => { + const request = mountRequest(); + const pending = deferred(); + request.handler.mockReturnValueOnce(pending.promise); + request.controller.begin(); + await request.select(); + await request.instruction("Make this title shorter."); + submit(request.container); + submit(request.container); + expect(request.handler).toHaveBeenCalledTimes(1); + pending.resolve({ admitted: true }); + await vi.waitFor(() => expect(request.container.querySelector("form")).toBeNull()); + expect(request.container.querySelector('[role="status"]')?.textContent).toContain( + "Request sent", + ); + const old = deferred(); + request.handler.mockReturnValueOnce(old.promise); + request.controller.begin(); + await request.select(); + await request.instruction("Old instructions."); + submit(request.container); + const signal = request.handler.mock.calls[1]?.[1]; + request.controller.reset(); + expect(signal?.aborted).toBe(true); + request.controller.begin(); + await request.select(); + await request.instruction("New instructions."); + old.resolve({ admitted: true }); + await Promise.resolve(); + await Promise.resolve(); + expect(request.container.querySelector("textarea")?.value).toBe("New instructions."); + expect(request.container.querySelector('[role="status"]')).toBeNull(); + }); + + it("retains the selected context and draft when PNG encoding exceeds the attachment bound", async () => { + const request = mountRequest(); + request.controller.begin(); + await request.select(); + await request.instruction("Keep these instructions after capture failure."); + vi.spyOn(HTMLCanvasElement.prototype, "toDataURL").mockReturnValueOnce( + "data:image/png;base64," + + "A".repeat(Math.ceil((DOCUMENT_CHANGE_CROP_MAX_BYTES + 10) / 3) * 4), + ); + submit(request.container); + await vi.waitFor(() => + expect(request.container.querySelector('[role="alert"]')?.textContent).toContain( + "could not be captured", + ), + ); + expect(request.container.querySelector("textarea")?.value).toContain("Keep these instructions"); + expect(request.container.querySelector(".selected-text")).not.toBeNull(); + expect(request.handler).not.toHaveBeenCalled(); + }); + + it("keeps the measured mobile popover within the visible stage and updates it on scroll", async () => { + vi.stubGlobal("innerWidth", 390); + const request = mountRequest({ + stageBounds: new DOMRect(10, 30, 370, 220), + canvasBounds: new DOMRect(20, 40, 350, 200), + }); + request.controller.begin(); + await request.select(0.99, 0.99); + const form = request.container.querySelector("form"); + const stage = request.container.querySelector(".stage"); + if (!form || !stage) { + throw new Error("Missing anchored mobile popover"); + } + const assertVisible = () => { + const bounds = form.getBoundingClientRect(); + expect(form.style.visibility).toBe("visible"); + expect(bounds.left).toBeGreaterThanOrEqual(10); + expect(bounds.right).toBeLessThanOrEqual(380); + expect(bounds.top).toBeGreaterThanOrEqual(30); + expect(bounds.bottom).toBeLessThanOrEqual(250); + }; + await vi.waitFor(assertVisible); + stage.scrollTop = 80; + stage.dispatchEvent(new Event("scroll")); + assertVisible(); + resizeObservers + .filter((observer) => !observer.disconnected) + .forEach((observer) => observer.notify()); + assertVisible(); + }); +}); diff --git a/ui/src/components/document-viewer-request.ts b/ui/src/components/document-viewer-request.ts new file mode 100644 index 0000000000..43b639310e --- /dev/null +++ b/ui/src/components/document-viewer-request.ts @@ -0,0 +1,523 @@ +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { html, nothing } from "lit"; +import { keyed } from "lit/directives/keyed.js"; +import { ref } from "lit/directives/ref.js"; +import { t } from "../i18n/index.ts"; +import type { DocumentSelectionTarget } from "./document-viewer-selection.ts"; + +export const DOCUMENT_CHANGE_TEXT_MAX = 1_024; +export const DOCUMENT_CHANGE_INSTRUCTION_MAX = 4_096; +export const DOCUMENT_CHANGE_CROP_MAX_BYTES = 1024 * 1024; +const CROP_MAX_PIXELS = 256_000; +const CROP_MAX_DIMENSION = 768; +const POINT_REGION_WIDTH = 0.35; +const POINT_REGION_HEIGHT = 0.18; +const POPOVER_MARGIN = 12; +const POPOVER_GAP = 8; +const POPOVER_WIDTH = 300; + +type DocumentChangeRegion = { x: number; y: number; width: number; height: number }; +export type DocumentChangeRequest = { + pageNumber: number; + region?: DocumentChangeRegion; + selectedText?: string; + instruction: string; + cropDataUrl?: string; +}; +export type DocumentChangeResult = { admitted: true } | { admitted: false; message?: string }; +export type DocumentChangeHandler = ( + request: DocumentChangeRequest, + signal: AbortSignal, +) => Promise; + +type SelectionPage = { + number: number; + canvas: HTMLCanvasElement; + targets?: readonly DocumentSelectionTarget[]; +}; +type RequestHost = { + requestUpdate(): void; + currentPage(): SelectionPage | undefined; + handler(): DocumentChangeHandler | undefined; +}; + +/** Selection stays local until the current document owner admits an explicit request. */ +export class DocumentViewerRequestController { + active = false; + busy = false; + private pageNumber = 0; + private region: DocumentChangeRegion | undefined; + private selectedText = ""; + private instruction = ""; + private error = ""; + private sent = false; + private drag: { pointerId: number; x: number; y: number; canvas: HTMLCanvasElement } | undefined; + private submission: AbortController | undefined; + private popover: HTMLFormElement | undefined; + private popoverObserver: ResizeObserver | undefined; + private selectionVersion = 0; + + constructor(private readonly host: RequestHost) {} + + begin(): void { + const page = this.host.currentPage(); + if (!page || !this.host.handler()) { + return; + } + this.active = true; + this.pageNumber = page.number; + this.sent = false; + this.host.requestUpdate(); + } + + private retireSelection(): void { + this.selectionVersion++; + this.submission?.abort(); + this.submission = undefined; + this.busy = false; + this.region = this.drag = undefined; + this.selectedText = this.error = ""; + this.releasePopover(); + } + + reset(): void { + this.retireSelection(); + this.active = this.sent = false; + this.instruction = ""; + this.host.requestUpdate(); + } + + dismissSelection(): void { + this.retireSelection(); + this.instruction = ""; + this.host.requestUpdate(); + } + + changePage(pageNumber: number): void { + if (this.active && this.pageNumber !== pageNumber) { + this.retireSelection(); + this.pageNumber = pageNumber; + this.host.requestUpdate(); + } + } + + private selectionPage(): SelectionPage | undefined { + const page = this.host.currentPage(); + return this.active && page?.number === this.pageNumber ? page : undefined; + } + + private point(event: PointerEvent, page: SelectionPage): { x: number; y: number } | undefined { + const bounds = page.canvas.getBoundingClientRect(); + if (!page.canvas.isConnected || bounds.width <= 0 || bounds.height <= 0) { + return undefined; + } + return { + x: Math.max(0, Math.min(1, (event.clientX - bounds.left) / bounds.width)), + y: Math.max(0, Math.min(1, (event.clientY - bounds.top) / bounds.height)), + }; + } + + pointerDown = (event: PointerEvent): void => { + const page = this.selectionPage(); + if (!page || this.busy || event.button !== 0 || !(event.currentTarget instanceof HTMLElement)) { + return; + } + const point = this.point(event, page); + if (!point) { + return; + } + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + this.drag = { ...point, pointerId: event.pointerId, canvas: page.canvas }; + this.region = undefined; + this.selectedText = this.error = ""; + this.host.requestUpdate(); + }; + + pointerMove = (event: PointerEvent): void => { + const drag = this.drag; + if (!drag || drag.pointerId !== event.pointerId) { + return; + } + const page = this.selectionPage(); + if (!page || page.canvas !== drag.canvas) { + this.pointerCancel(); + return; + } + const point = this.point(event, page); + if (!point) { + return; + } + this.region = { + x: Math.min(drag.x, point.x), + y: Math.min(drag.y, point.y), + width: Math.abs(point.x - drag.x), + height: Math.abs(point.y - drag.y), + }; + this.host.requestUpdate(); + }; + + pointerUp = (event: PointerEvent): void => { + const drag = this.drag; + if (!drag || drag.pointerId !== event.pointerId) { + return; + } + const page = this.selectionPage(); + if (!page || page.canvas !== drag.canvas) { + this.pointerCancel(); + return; + } + const point = this.point(event, page); + if (!point) { + this.pointerCancel(); + return; + } + this.pointerMove(event); + this.drag = undefined; + const bounds = page.canvas.getBoundingClientRect(); + const distance = Math.hypot( + (point.x - drag.x) * bounds.width, + (point.y - drag.y) * bounds.height, + ); + if (distance < 3) { + let target: DocumentSelectionTarget | undefined; + for (const candidate of page.targets ?? []) { + const region = candidate.region; + if ( + point.x >= region.x && + point.x <= region.x + region.width && + point.y >= region.y && + point.y <= region.y + region.height && + (!target || region.width * region.height < target.region.width * target.region.height) + ) { + target = candidate; + } + } + if (target) { + this.region = { ...target.region }; + this.selectedText = truncateUtf16Safe(target.text, DOCUMENT_CHANGE_TEXT_MAX); + } else { + this.region = { + x: Math.max(0, Math.min(1 - POINT_REGION_WIDTH, point.x - POINT_REGION_WIDTH / 2)), + y: Math.max(0, Math.min(1 - POINT_REGION_HEIGHT, point.y - POINT_REGION_HEIGHT / 2)), + width: POINT_REGION_WIDTH, + height: POINT_REGION_HEIGHT, + }; + } + } else if (this.region) { + // Preserve real drags, including a thin horizontal/vertical gesture, with + // a one-CSS-pixel minimum so the crop always has a nonempty source area. + const width = Math.min(1, Math.max(this.region.width, 1 / bounds.width)); + const height = Math.min(1, Math.max(this.region.height, 1 / bounds.height)); + this.region = { + x: Math.min(this.region.x, 1 - width), + y: Math.min(this.region.y, 1 - height), + width, + height, + }; + } + this.selectionVersion++; + this.host.requestUpdate(); + }; + + pointerCancel = (): void => { + if (this.drag) { + this.region = undefined; + } + this.drag = undefined; + this.host.requestUpdate(); + }; + + private releasePopover(): void { + this.popoverObserver?.disconnect(); + this.popoverObserver = undefined; + window.removeEventListener("resize", this.positionPopover); + window.removeEventListener("scroll", this.positionPopover, true); + this.popover = undefined; + } + + private bindPopover = (element?: Element): void => { + this.releasePopover(); + if (!(element instanceof HTMLFormElement)) { + return; + } + const stage = this.selectionPage()?.canvas.closest(".stage"); + if (!stage) { + return; + } + this.popover = element; + this.popoverObserver = new ResizeObserver(this.positionPopover); + this.popoverObserver.observe(element); + this.popoverObserver.observe(stage); + window.addEventListener("resize", this.positionPopover); + window.addEventListener("scroll", this.positionPopover, true); + queueMicrotask(() => { + if (this.popover === element && element.isConnected) { + this.positionPopover(); + element.querySelector("textarea")?.focus({ preventScroll: true }); + } + }); + }; + + private positionPopover = (): void => { + const form = this.popover; + const page = this.selectionPage(); + const region = this.region; + const stage = page?.canvas.closest(".stage"); + if (!form?.isConnected || !page?.canvas.isConnected || !region || !stage) { + return; + } + const stageBounds = stage.getBoundingClientRect(); + const canvasBounds = page.canvas.getBoundingClientRect(); + let left = Math.max(stageBounds.left + stage.clientLeft, 0); + let top = Math.max(stageBounds.top + stage.clientTop, 0); + let right = Math.min( + stageBounds.left + stage.clientLeft + stage.clientWidth, + window.innerWidth, + ); + let bottom = Math.min( + stageBounds.top + stage.clientTop + stage.clientHeight, + window.innerHeight, + ); + // Intersect clipping ancestors too: inline previews may be in a scrolling + // chat pane, with a visible stage smaller than its full document rectangle. + let ancestor: Element | null = stage.parentElement; + while (ancestor) { + const style = getComputedStyle(ancestor); + const bounds = ancestor.getBoundingClientRect(); + if (/(auto|scroll|hidden|clip)/u.test(style.overflowX)) { + left = Math.max(left, bounds.left + ancestor.clientLeft); + right = Math.min(right, bounds.left + ancestor.clientLeft + ancestor.clientWidth); + } + if (/(auto|scroll|hidden|clip)/u.test(style.overflowY)) { + top = Math.max(top, bounds.top + ancestor.clientTop); + bottom = Math.min(bottom, bounds.top + ancestor.clientTop + ancestor.clientHeight); + } + const root = ancestor.getRootNode(); + ancestor = ancestor.parentElement ?? (root instanceof ShadowRoot ? root.host : null); + } + if (right <= left || bottom <= top) { + form.style.visibility = "hidden"; + return; + } + const marginX = Math.min(POPOVER_MARGIN, (right - left) / 4); + const marginY = Math.min(POPOVER_MARGIN, (bottom - top) / 4); + left += marginX; + right -= marginX; + top += marginY; + bottom -= marginY; + const width = Math.min(POPOVER_WIDTH, right - left); + const maxHeight = bottom - top; + form.style.setProperty("--request-popover-width", `${width}px`); + form.style.setProperty("--request-popover-max-height", `${maxHeight}px`); + const height = Math.min(form.getBoundingClientRect().height, maxHeight); + const selectionLeft = canvasBounds.left + region.x * canvasBounds.width; + const selectionRight = selectionLeft + region.width * canvasBounds.width; + const selectionTop = canvasBounds.top + region.y * canvasBounds.height; + const selectionBottom = selectionTop + region.height * canvasBounds.height; + const beside = selectionRight + POPOVER_GAP + width <= right; + const preferredLeft = beside ? selectionRight + POPOVER_GAP : selectionLeft; + let preferredTop = beside ? selectionTop : selectionBottom + POPOVER_GAP; + if (preferredTop + height > bottom) { + preferredTop = selectionTop - height - POPOVER_GAP; + } + const x = Math.max(left, Math.min(right - width, preferredLeft)); + const y = Math.max(top, Math.min(bottom - height, preferredTop)); + form.style.setProperty( + "--request-popover-left", + `${x - stageBounds.left - stage.clientLeft + stage.scrollLeft}px`, + ); + form.style.setProperty( + "--request-popover-top", + `${y - stageBounds.top - stage.clientTop + stage.scrollTop}px`, + ); + form.style.visibility = "visible"; + }; + + renderOverlay() { + const region = this.region; + return this.active && region + ? html`
` + : nothing; + } + + renderStatus() { + return this.sent + ? html`
+ ${t("chat.documentViewer.requestSent")} +
` + : nothing; + } + + renderPopover() { + if (!this.selectionPage() || !this.region || this.drag) { + return nothing; + } + return keyed( + this.selectionVersion, + html``, + ); + } + + private captureCrop(canvas: HTMLCanvasElement, region: DocumentChangeRegion): string { + const width = Math.max(1, Math.round(canvas.width * region.width)); + const height = Math.max(1, Math.round(canvas.height * region.height)); + const scale = Math.min( + 1, + Math.sqrt(CROP_MAX_PIXELS / (width * height)), + CROP_MAX_DIMENSION / width, + CROP_MAX_DIMENSION / height, + ); + const crop = document.createElement("canvas"); + try { + crop.width = Math.max(1, Math.floor(width * scale)); + crop.height = Math.max(1, Math.floor(height * scale)); + const context = crop.getContext("2d"); + if (!context) { + throw new Error("Canvas unavailable"); + } + context.drawImage( + canvas, + canvas.width * region.x, + canvas.height * region.y, + width, + height, + 0, + 0, + crop.width, + crop.height, + ); + const data = crop.toDataURL("image/png"); + const encoded = data.slice("data:image/png;base64,".length); + const bytes = + Math.floor((encoded.length * 3) / 4) - + (encoded.endsWith("==") ? 2 : encoded.endsWith("=") ? 1 : 0); + if (!data.startsWith("data:image/png;base64,") || bytes > DOCUMENT_CHANGE_CROP_MAX_BYTES) { + throw new Error("Crop exceeds attachment bound"); + } + return data; + } finally { + crop.width = crop.height = 0; + } + } + + private submit = async (event: Event): Promise => { + event.preventDefault(); + const page = this.host.currentPage(); + const handler = this.host.handler(); + const instruction = this.instruction.trim(); + if ( + !this.active || + this.busy || + !page || + page.number !== this.pageNumber || + !handler || + !instruction || + (!this.region && !this.selectedText) + ) { + return; + } + this.error = ""; + let cropDataUrl: string | undefined; + if (this.region) { + try { + cropDataUrl = this.captureCrop(page.canvas, this.region); + } catch { + this.error = t("chat.documentViewer.requestCropError"); + this.host.requestUpdate(); + return; + } + } + const submission = new AbortController(); + this.submission = submission; + this.busy = true; + this.host.requestUpdate(); + try { + const result = await handler( + { + pageNumber: page.number, + region: this.region, + selectedText: this.selectedText || undefined, + instruction, + cropDataUrl, + }, + submission.signal, + ); + if (this.submission !== submission) { + return; + } + if (result.admitted) { + this.reset(); + this.sent = true; + } else { + this.error = result.message + ? truncateUtf16Safe(result.message, 500) + : t("chat.documentViewer.requestUnavailable"); + } + } catch { + if (this.submission === submission) { + this.error = t("chat.documentViewer.requestUnavailable"); + } + } finally { + if (this.submission === submission) { + this.submission = undefined; + this.busy = false; + } + this.host.requestUpdate(); + } + }; +} diff --git a/ui/src/components/document-viewer-selection.test.ts b/ui/src/components/document-viewer-selection.test.ts new file mode 100644 index 0000000000..a3322d02cb --- /dev/null +++ b/ui/src/components/document-viewer-selection.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, it } from "vitest"; +import { createDocumentSelectionTargets } from "./document-viewer-selection.ts"; +import type { PDFPageProxy } from "./pdf-document.runtime.ts"; + +type Viewport = ReturnType; +type Content = Awaited>; +type TextItem = Extract; + +function viewport(overrides: Partial = {}): Viewport { + return { + viewBox: [0, 0, 200, 200], + userUnit: 1, + scale: 1, + rotation: 0, + offsetX: 0, + offsetY: 0, + width: 200, + height: 200, + transform: [1, 0, 0, -1, 0, 200], + rawDims: { pageWidth: 200, pageHeight: 200, pageX: 0, pageY: 0 }, + clone() { + throw new Error("Geometry must use the supplied viewport"); + }, + convertToViewportPoint() { + throw new Error("Geometry must use the supplied transform"); + }, + convertToPdfPoint() { + throw new Error("Geometry must not invert the viewport"); + }, + ...overrides, + }; +} + +function item(overrides: Partial = {}): TextItem { + return { + str: "Heading", + dir: "ltr", + transform: [20, 0, 0, 20, 50, 100], + width: 60, + height: 20, + fontName: "heading", + hasEOL: false, + ...overrides, + }; +} + +function content(items: Content["items"] = [item()], style = {}): Content { + return { + items, + styles: { + heading: { ascent: 0.8, descent: -0.2, vertical: false, fontFamily: "sans-serif", ...style }, + }, + lang: "en", + }; +} + +function expectRegion(contentValue: Content, viewportValue: Viewport, expected: number[]) { + const targets = createDocumentSelectionTargets(contentValue, viewportValue); + expect(targets).toHaveLength(1); + const { x, y, width, height } = targets[0]!.region; + [x, y, width, height].forEach((value, index) => expect(value).toBeCloseTo(expected[index]!, 10)); + expect(targets[0]!.text).toBe("Heading"); +} + +describe("document text selection geometry", () => { + it("places a 20-unit font box 16 units above its baseline, with the recorded text advance", () => { + // PDF baseline (50,100), ascent16/descent4, advance60 on a200-square page. + expectRegion(content(), viewport(), [0.25, 0.42, 0.3, 0.1]); + }); + + it.each([ + { rotation: 90, transform: [0, 1, 1, 0, 0, 0], region: [0.48, 0.25, 0.1, 0.3] }, + { rotation: 180, transform: [-1, 0, 0, 1, 200, 0], region: [0.45, 0.48, 0.3, 0.1] }, + { rotation: 270, transform: [0, -1, -1, 0, 200, 200], region: [0.42, 0.45, 0.1, 0.3] }, + ])("rotates the baseline box with a $rotation-degree page", ({ rotation, transform, region }) => { + expectRegion(content(), viewport({ rotation, transform }), region); + }); + + it("normalizes a quarter-turn against the swapped dimensions of a rectangular page", () => { + // A200x300 PDF page becomes300x200; the visible box is(96,50)–(116,110). + expectRegion( + content(), + viewport({ + viewBox: [0, 0, 200, 300], + rotation: 90, + width: 300, + height: 200, + transform: [0, 1, 1, 0, 0, 0], + }), + [0.32, 0.25, 1 / 15, 0.3], + ); + }); + + it("uses the viewport's negative crop origin and userUnit exactly once", () => { + expectRegion( + content([item({ transform: [20, 0, 0, 20, -50, 50] })]), + viewport({ + viewBox: [-100, -50, 100, 150], + scale: 1.5, + userUnit: 2, + width: 600, + height: 600, + transform: [3, 0, 0, -3, 300, 450], + }), + [0.25, 0.42, 0.3, 0.1], + ); + }); + + it("honors rotated text and vertical-writing advance", () => { + expectRegion( + content([item({ transform: [0, 20, -20, 0, 100, 50] })]), + viewport(), + [0.42, 0.45, 0.1, 0.3], + ); + expectRegion( + content([item({ height: 80 })], { vertical: true }), + viewport(), + [0.23, 0.5, 0.1, 0.4], + ); + }); + + it("uses descent metrics when ascent is absent, otherwise the documented 0.8 fallback", () => { + expectRegion( + content(undefined, { ascent: 0, descent: -0.3 }), + viewport(), + [0.25, 0.43, 0.3, 0.1], + ); + expectRegion(content(undefined, { ascent: 0, descent: 0 }), viewport(), [0.25, 0.42, 0.3, 0.1]); + }); + + it("clips partially visible boxes and discards empty, outside or invalid entries", () => { + const source = content([ + { type: "beginMarkedContent", id: "marker" }, + item({ str: " " }), + item({ transform: [20, 0, 0, 20, 250, 100] }), + item({ width: 0 }), + item({ width: Number.POSITIVE_INFINITY }), + item({ height: Number.NaN }), + item({ transform: [0, 0, 0, 0, 50, 100] }), + item({ transform: [20, 0, 0, 20, Number.NaN, 100] }), + item({ transform: [20, 0, 0, 20, -20, 100] }), + ]); + expectRegion(source, viewport(), [0, 0.42, 0.2, 0.1]); + expect(createDocumentSelectionTargets(content(), viewport({ width: 0 }))).toEqual([]); + expect( + createDocumentSelectionTargets( + content(), + viewport({ transform: [1, 0, 0, -1, 0, Number.NaN] }), + ), + ).toEqual([]); + }); + + it("bounds targets and UTF16 text without splitting a surrogate pair", () => { + const source = content( + Array.from({ length: 2_050 }, () => item({ str: ` ${"x".repeat(1_023)}😀tail ` })), + ); + const targets = createDocumentSelectionTargets(source, viewport()); + expect(targets).toHaveLength(2_048); + expect(targets.every((target) => target.text === "x".repeat(1_023))).toBe(true); + }); + + it("stops scanning after 8192 marked-content items even when no target was emitted", () => { + const markers = Array.from({ length: 8_192 }, () => ({ + type: "beginMarkedContent", + id: "marker", + })); + expect(createDocumentSelectionTargets(content([...markers, item()]), viewport())).toEqual([]); + }); +}); diff --git a/ui/src/components/document-viewer-selection.ts b/ui/src/components/document-viewer-selection.ts new file mode 100644 index 0000000000..1fa7fcf0b6 --- /dev/null +++ b/ui/src/components/document-viewer-selection.ts @@ -0,0 +1,101 @@ +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import type { PDFPageProxy } from "./pdf-document.runtime.ts"; + +export type DocumentSelectionTarget = { + region: { x: number; y: number; width: number; height: number }; + text: string; +}; + +type Matrix = [number, number, number, number, number, number]; + +function isFiniteMatrix(value: unknown): value is Matrix { + return ( + Array.isArray(value) && + value.length === 6 && + value.every((entry) => typeof entry === "number" && Number.isFinite(entry)) + ); +} + +/** PDF font-metric boxes, not glyph outlines or native presentation objects. */ +export function createDocumentSelectionTargets( + content: Awaited>, + viewport: ReturnType, +): DocumentSelectionTarget[] { + const targets: DocumentSelectionTarget[] = []; + if ( + !Number.isFinite(viewport.width) || + !Number.isFinite(viewport.height) || + viewport.width <= 0 || + viewport.height <= 0 || + !isFiniteMatrix(viewport.transform) + ) { + return targets; + } + const [a, b, c, d, e, f] = viewport.transform; + // PageViewport already includes page rotation, viewBox origin and userUnit. + const scale = Math.hypot(a, b); + if (!Number.isFinite(scale) || scale <= 0) { + return targets; + } + // Bound scanned work as well as emitted boxes: marked content or invalid text + // in a highly compressed page must not consume an unbounded geometry budget. + for (const item of content.items.slice(0, 8_192)) { + if (!("str" in item) || !item.str.trim() || !isFiniteMatrix(item.transform)) { + continue; + } + const [ta, tb, tc, td, tx, ty] = item.transform; + const style = content.styles[item.fontName]; + const vertical = style?.vertical ?? false; + const advance = (vertical ? item.height : item.width) * scale; + const fontHeight = Math.hypot(a * tc + c * td, b * tc + d * td); + if ( + !Number.isFinite(item.width) || + !Number.isFinite(item.height) || + !Number.isFinite(advance) || + advance <= 0 || + !Number.isFinite(fontHeight) || + fontHeight <= 0 + ) { + continue; + } + // TextLayer uses this baseline angle and vertical-writing adjustment. Its + // browser-font measurements are unavailable here; retain the PDF metrics + // fallback (ascent, then descent, then 0.8) without a DOM text overlay. + const angle = Math.atan2(b * ta + d * tb, a * ta + c * tb) + (vertical ? Math.PI / 2 : 0); + const fontAscent = style?.ascent; + const fontDescent = style?.descent; + const ascent = + typeof fontAscent === "number" && Number.isFinite(fontAscent) && fontAscent > 0 + ? fontAscent + : typeof fontDescent === "number" && Number.isFinite(fontDescent) && fontDescent < 0 + ? 1 + fontDescent + : 0.8; + const sin = Math.sin(angle); + const cos = Math.cos(angle); + const left = a * tx + c * ty + e + fontHeight * ascent * sin; + const top = b * tx + d * ty + f - fontHeight * ascent * cos; + const corners: [number, number][] = [ + [left, top], + [left + advance * cos, top + advance * sin], + [left - fontHeight * sin, top + fontHeight * cos], + [left + advance * cos - fontHeight * sin, top + advance * sin + fontHeight * cos], + ]; + const xs = corners.map(([x]) => x); + const ys = corners.map(([, y]) => y); + const x = Math.max(0, Math.min(...xs) / viewport.width); + const y = Math.max(0, Math.min(...ys) / viewport.height); + const right = Math.min(1, Math.max(...xs) / viewport.width); + const bottom = Math.min(1, Math.max(...ys) / viewport.height); + if (![x, y, right, bottom].every(Number.isFinite) || right <= x || bottom <= y) { + continue; + } + targets.push({ + region: { x, y, width: right - x, height: bottom - y }, + text: truncateUtf16Safe(item.str.trim(), 1_024), + }); + if (targets.length === 2_048) { + break; + } + } + return targets; +} diff --git a/ui/src/components/document-viewer.styles.ts b/ui/src/components/document-viewer.styles.ts new file mode 100644 index 0000000000..a0f0bcb694 --- /dev/null +++ b/ui/src/components/document-viewer.styles.ts @@ -0,0 +1,380 @@ +import { css } from "lit"; + +export const documentViewerStyles = css` + :host { + display: block; + min-width: 0; + width: 100%; + } + .viewer { + display: flex; + flex-direction: column; + min-width: 0; + overflow: hidden; + color: var(--text, #222); + } + .header { + min-width: 0; + padding: 12px; + border-bottom: 1px solid var(--border, #ccc); + } + .document-title { + display: block; + overflow: hidden; + font-size: 14px; + text-overflow: ellipsis; + white-space: nowrap; + } + .toolbar { + display: grid; + grid-template-columns: 1fr auto; + align-items: center; + gap: 8px; + padding: 8px 0; + border-top: 1px solid var(--border, #ccc); + } + .controls, + .pagination, + .option-controls { + display: flex; + align-items: center; + gap: 6px; + } + .pagination { + justify-content: center; + min-width: 0; + } + .option-controls { + flex-wrap: wrap; + } + .request-changes { + display: inline-flex; + align-items: center; + gap: 5px; + white-space: nowrap; + } + .request-icon, + .request-icon svg { + display: block; + width: 16px; + height: 16px; + } + button, + input, + textarea, + summary { + box-sizing: border-box; + color: inherit; + font: inherit; + font-size: 12px; + } + button { + min-height: 32px; + padding: 4px 8px; + border: 0; + border-radius: 5px; + background: transparent; + cursor: pointer; + } + button:hover, + .preview-options > summary:hover { + background: var(--bg-hover, #eee); + } + button:disabled { + opacity: 0.4; + cursor: default; + } + button[aria-pressed="true"], + button[aria-expanded="true"] { + background: var(--bg-hover, #eee); + color: var(--accent, #159b93); + } + button:focus-visible, + input:focus-visible, + textarea:focus-visible, + summary:focus-visible, + .stage:focus-visible { + outline: 2px solid var(--accent, #159b93); + outline-offset: 2px; + } + .position, + .zoom-level { + font-size: 12px; + font-variant-numeric: tabular-nums; + white-space: nowrap; + } + .position { + min-width: 36px; + text-align: center; + } + .page-unit { + font-size: 12px; + color: var(--muted, #777); + } + .zoom-level { + min-width: 34px; + text-align: center; + } + .page-layout { + display: flex; + height: min(65dvh, 640px); + min-height: 240px; + min-width: 0; + } + .stage { + position: relative; + box-sizing: border-box; + flex: 1; + height: 100%; + min-height: 0; + min-width: 0; + overflow: auto; + scrollbar-gutter: stable; + padding: 16px; + border-radius: 4px; + background: #d5d7da; + overscroll-behavior: contain; + } + .current-page { + display: block; + margin: 0; + background: #fff; + box-shadow: 0 2px 12px #0003; + } + .current-page[aria-hidden="true"] { + visibility: hidden; + } + openclaw-document-contents { + flex: 0 0 132px; + border-right: 1px solid var(--border, #ccc); + } + .page-surface { + position: relative; + width: fit-content; + margin: 0 auto; + } + .selection-active { + cursor: crosshair; + touch-action: none; + user-select: none; + } + .selection-region { + position: absolute; + pointer-events: none; + box-sizing: border-box; + border: 2px solid var(--accent, #159b93); + background: #159b9326; + } + .selection-hint { + padding: 0 8px 6px; + color: var(--muted, #777); + font-size: 12px; + } + .request-popover { + position: absolute; + z-index: 3; + left: var(--request-popover-left, 12px); + top: var(--request-popover-top, 12px); + width: var(--request-popover-width, min(300px, calc(100% - 24px))); + max-height: var(--request-popover-max-height); + overflow: auto; + display: flex; + flex-direction: column; + gap: 8px; + margin: 0; + padding: 12px; + box-sizing: border-box; + border: 1px solid var(--border, #ccc); + border-radius: 8px; + background: var(--bg, #fff); + color: var(--text, #222); + box-shadow: 0 6px 24px #0003; + font-size: 12px; + } + .request-popover-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + } + .request-popover-header strong { + font-weight: 500; + } + .request-popover-header button { + min-height: 24px; + padding: 0 5px; + } + .request-popover label { + display: flex; + flex-direction: column; + gap: 4px; + } + .request-instruction { + display: block; + width: 100%; + min-height: 58px; + padding: 7px 8px; + resize: vertical; + border: 1px solid var(--border, #ccc); + border-radius: 5px; + background: var(--bg, #fff); + } + .request-popover .controls { + justify-content: flex-end; + } + .request-popover button[type="submit"] { + align-self: flex-end; + background: var(--accent, #159b93); + color: var(--accent-foreground, #fff); + } + .selected-text { + margin: 0; + max-height: 38px; + overflow: auto; + color: var(--muted, #777); + font-size: 11px; + white-space: pre-wrap; + overflow-wrap: anywhere; + } + .request-error { + color: var(--danger, #b42318); + } + .request-result { + padding: 5px 8px; + font-size: 12px; + } + .notice { + position: absolute; + z-index: 1; + inset: 0 auto auto 0; + padding: 12px; + background: var(--bg, #fff); + font-size: 13px; + } + .preview-options { + position: relative; + } + .preview-options > summary { + display: grid; + place-items: center; + min-width: 30px; + min-height: 32px; + border-radius: 5px; + list-style: none; + cursor: pointer; + } + .preview-options > summary::-webkit-details-marker { + display: none; + } + .options-panel { + position: absolute; + z-index: 4; + right: 0; + bottom: calc(100% + 6px); + width: min(320px, calc(100vw - 72px)); + max-height: 55dvh; + overflow: auto; + padding: 12px; + border: 1px solid var(--border, #ccc); + border-radius: 8px; + background: var(--bg, #fff); + box-shadow: 0 6px 24px #0003; + } + .text-alternative { + margin-top: 10px; + font-size: 12px; + } + .text-alternative > summary { + cursor: pointer; + } + .page-text { + max-height: 140px; + overflow: auto; + padding-top: 8px; + white-space: pre-wrap; + overflow-wrap: anywhere; + } + .option-note { + margin: 10px 0 0; + color: var(--muted, #777); + font-size: 11px; + } + openclaw-modal-dialog { + --openclaw-modal-width: calc(100vw - 32px); + --openclaw-modal-max-width: calc(100vw - 32px); + --openclaw-modal-max-height: calc(100dvh - 32px); + } + .expanded { + width: calc(100vw - 32px); + height: calc(100dvh - 32px); + background: var(--bg, #fff); + } + .expanded .toolbar { + padding: 8px; + } + .expanded .page-layout { + flex: 1; + height: auto; + min-height: 0; + } + .expanded .stage { + height: auto; + } + @media (max-width: 600px) { + .page-layout { + flex-direction: column; + height: 60dvh; + } + openclaw-document-contents { + flex: 0 0 120px; + border-right: 0; + border-bottom: 1px solid var(--border, #ccc); + } + .stage { + height: auto; + padding: 12px; + } + .toolbar { + gap: 4px; + } + .pagination, + .controls { + gap: 2px; + } + .toolbar button { + padding: 4px 5px; + } + .request-instruction { + font-size: 16px; + } + openclaw-modal-dialog { + --openclaw-modal-width: calc(100vw - 16px); + --openclaw-modal-max-width: calc(100vw - 16px); + --openclaw-modal-max-height: calc(100dvh - 16px); + } + .expanded { + width: calc(100vw - 16px); + height: calc(100dvh - 16px); + } + } + @media (max-width: 420px) { + .toolbar { + grid-template-columns: 1fr 1fr; + } + .pagination { + grid-row: 1; + grid-column: 1 / -1; + } + .toolbar > .controls { + grid-column: 1 / -1; + justify-self: end; + } + } + @media (pointer: coarse) { + .toolbar button, + .preview-options > summary, + .request-popover button { + min-height: 44px; + } + } +`; diff --git a/ui/src/components/document-viewer.test.ts b/ui/src/components/document-viewer.test.ts new file mode 100644 index 0000000000..c48a12b3c8 --- /dev/null +++ b/ui/src/components/document-viewer.test.ts @@ -0,0 +1,539 @@ +/* @vitest-environment jsdom */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { settleLitElement } from "../test-helpers/lit-settle.ts"; +import { installDialogPolyfill } from "../test-helpers/modal-dialog.ts"; +import type { PDFPageProxy } from "./pdf-document.runtime.ts"; +import "./document-viewer.ts"; + +type TestText = Awaited>; +type TestPage = { + getViewport: ReturnType< + typeof vi.fn< + (options: { scale: number }) => { width: number; height: number; transform: number[] } + > + >; + getTextContent: ReturnType Promise>>; + cleanup: ReturnType; +}; +type TestRenderTask = { promise: Promise; cancel: ReturnType }; +type TestHandle = { promise: Promise; destroy: () => Promise }; + +const runtime = vi.hoisted(() => ({ + openPdfDocument: + vi.fn< + ( + bytes: Uint8Array | ArrayBuffer, + onFailure: (kind: "password" | "unavailable") => void, + ) => TestHandle + >(), + renderPdfPage: + vi.fn< + ( + page: TestPage, + canvas: HTMLCanvasElement, + viewport: { width: number; height: number }, + ) => TestRenderTask + >(), +})); + +const resizeObservers: TestResizeObserver[] = []; + +class TestResizeObserver { + target: Element | undefined; + constructor(private readonly callback: () => void) { + resizeObservers.push(this); + } + observe(target: Element): void { + this.target = target; + } + unobserve(): void { + this.target = undefined; + } + disconnect(): void { + this.target = undefined; + } + notify(): void { + if (this.target) { + this.callback(); + } + } +} + +// Pixel/font/worker integration belongs to the real browser scenario. Only the +// narrow PDF runtime is replaced here to control otherwise nondeterministic races. +vi.mock("./pdf-document.runtime.ts", () => runtime); + +function deferred() { + let resolve: (value: T) => void = () => {}; + let reject: (reason: Error) => void = () => {}; + const promise = new Promise((accept, fail) => { + resolve = accept; + reject = fail; + }); + return { promise, resolve, reject }; +} + +function createPdf(pageCount = 2) { + const pages: TestPage[] = Array.from({ length: pageCount }, (_, index) => ({ + getViewport: vi.fn(({ scale }: { scale: number }) => ({ + width: 612 * scale, + height: 792 * scale, + transform: [scale, 0, 0, -scale, 0, 792 * scale], + })), + getTextContent: vi.fn(async () => pageText(`Page ${index + 1} text`)), + cleanup: vi.fn(), + })); + return { + pages, + numPages: pageCount, + getPage: vi.fn(async (number: number) => { + const page = pages[number - 1]; + if (!page) { + throw new Error("requested page is outside the test document"); + } + return page; + }), + }; +} + +function pageText(text: string): TestText { + return { + items: [ + { + str: text, + hasEOL: true, + dir: "ltr", + transform: [16, 0, 0, 16, 50, 700], + width: 100, + height: 16, + fontName: "body", + }, + ], + styles: { body: { ascent: 0.8, descent: -0.2, vertical: false, fontFamily: "sans-serif" } }, + lang: "en", + }; +} + +function createHandle() { + const loading = deferred>(); + const destroy = vi.fn(async () => {}); + return { loading, destroy, handle: { promise: loading.promise, destroy } }; +} + +async function mount() { + const viewer = document.createElement("openclaw-document-viewer"); + viewer.bytes = new Uint8Array([1]); + viewer.documentTitle = "Owned report.pdf"; + const loaded = vi.fn(); + const failed = vi.fn(); + viewer.addEventListener("document-viewer-load", loaded); + viewer.addEventListener("document-viewer-error", failed); + document.body.append(viewer); + await vi.waitFor(() => expect(runtime.openPdfDocument).toHaveBeenCalledTimes(1)); + const stage = viewer.shadowRoot?.querySelector(".stage"); + if (!stage) { + throw new Error("document viewer did not mount its page stage"); + } + // jsdom has no layout/adopted stylesheet measurements. These fixed stage + // dimensions merely allow the production render lifecycle to reach its runtime. + stage.style.padding = "16px"; + Object.defineProperties(stage, { + clientWidth: { configurable: true, value: 800 }, + clientHeight: { configurable: true, value: 600 }, + }); + await settleLitElement(viewer); + return { viewer, loaded, failed }; +} + +async function ready(viewer: HTMLElementTagNameMap["openclaw-document-viewer"], page = 1) { + await vi.waitFor(() => { + expect(viewer.getAttribute("data-render-state")).toBe("ready"); + expect(viewer.shadowRoot?.querySelector("canvas")?.getAttribute("data-rendered-page")).toBe( + String(page), + ); + }); +} + +beforeEach(() => { + vi.resetAllMocks(); + resizeObservers.length = 0; + runtime.renderPdfPage.mockImplementation(() => ({ promise: Promise.resolve(), cancel: vi.fn() })); + vi.stubGlobal("ResizeObserver", TestResizeObserver); +}); + +afterEach(() => { + document.body.replaceChildren(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +describe("document viewer asynchronous ownership", () => { + it("destroys a replaced load and ignores its late document and failure callback", async () => { + const first = createHandle(); + const second = createHandle(); + runtime.openPdfDocument.mockReturnValueOnce(first.handle).mockReturnValueOnce(second.handle); + const { viewer, loaded, failed } = await mount(); + viewer.bytes = new Uint8Array([2]); + await vi.waitFor(() => expect(runtime.openPdfDocument).toHaveBeenCalledTimes(2)); + expect(first.destroy).toHaveBeenCalledTimes(1); + const currentPdf = createPdf(3); + second.loading.resolve(currentPdf); + await ready(viewer); + const oldPdf = createPdf(); + first.loading.resolve(oldPdf); + runtime.openPdfDocument.mock.calls[0]![1]("password"); + await settleLitElement(viewer); + expect(oldPdf.getPage).not.toHaveBeenCalled(); + expect(viewer.shadowRoot?.querySelector("canvas")?.getAttribute("data-page-count")).toBe("3"); + expect(viewer.shadowRoot?.querySelector('[role="alert"]')).toBeNull(); + expect(loaded).toHaveBeenCalledTimes(1); + expect(failed).not.toHaveBeenCalled(); + viewer.remove(); + expect(second.destroy).toHaveBeenCalledTimes(1); + }); + + it("destroys a disconnected pending load without publishing its late result", async () => { + const pending = createHandle(); + runtime.openPdfDocument.mockReturnValueOnce(pending.handle); + const { viewer, loaded, failed } = await mount(); + viewer.remove(); + expect(pending.destroy).toHaveBeenCalledTimes(1); + const pdf = createPdf(); + pending.loading.resolve(pdf); + await settleLitElement(viewer); + expect(pdf.getPage).not.toHaveBeenCalled(); + expect(loaded).not.toHaveBeenCalled(); + expect(failed).not.toHaveBeenCalled(); + }); + + it.each(["replacement", "disconnect"] as const)( + "cancels an active render on %s without late publication", + async (action) => { + const pending = createHandle(); + const replacement = createHandle(); + const rendering = deferred(); + const cancel = vi.fn(); + runtime.openPdfDocument + .mockReturnValueOnce(pending.handle) + .mockReturnValueOnce(replacement.handle); + runtime.renderPdfPage.mockReturnValueOnce({ promise: rendering.promise, cancel }); + const { viewer, loaded, failed } = await mount(); + const pdf = createPdf(); + pending.loading.resolve(pdf); + await vi.waitFor(() => expect(runtime.renderPdfPage).toHaveBeenCalledTimes(1)); + const canvas = viewer.shadowRoot?.querySelector("canvas"); + expect(canvas?.width).toBeGreaterThan(0); + if (action === "replacement") { + viewer.bytes = new Uint8Array([2]); + await vi.waitFor(() => expect(runtime.openPdfDocument).toHaveBeenCalledTimes(2)); + } else { + viewer.remove(); + } + expect(cancel).toHaveBeenCalledTimes(1); + expect(pending.destroy).toHaveBeenCalledTimes(1); + expect(canvas?.width).toBe(0); + if (action === "replacement") { + replacement.loading.resolve(createPdf(3)); + await ready(viewer); + } + rendering.resolve(); + await settleLitElement(viewer); + expect(pdf.pages[0]?.getTextContent).not.toHaveBeenCalled(); + expect(loaded).toHaveBeenCalledTimes(action === "replacement" ? 1 : 0); + if (action === "replacement") { + expect(viewer.shadowRoot?.querySelector("canvas")?.getAttribute("data-page-count")).toBe( + "3", + ); + } + expect(failed).not.toHaveBeenCalled(); + }, + ); + + it.each(["password", "parse"] as const)( + "retires a failed %s load with a safe visible outcome", + async (failure) => { + const document = createHandle(); + runtime.openPdfDocument.mockReturnValueOnce(document.handle); + const { viewer, loaded, failed } = await mount(); + if (failure === "password") { + runtime.openPdfDocument.mock.calls[0]![1]("password"); + } else { + document.loading.reject(new Error("private parser diagnostic")); + } + await vi.waitFor(() => expect(viewer.getAttribute("data-render-state")).toBe("error")); + await settleLitElement(viewer); + const text = viewer.shadowRoot?.querySelector('[role="alert"]')?.textContent; + expect(text).toContain( + failure === "password" ? "requires a password" : "could not be displayed", + ); + expect(text).not.toContain("private parser diagnostic"); + expect(document.destroy).toHaveBeenCalledTimes(1); + expect(failed).toHaveBeenCalledTimes(1); + expect(loaded).not.toHaveBeenCalled(); + }, + ); + + it("never publishes an older page whose text completes after a newer page", async () => { + const document = createHandle(); + runtime.openPdfDocument.mockReturnValueOnce(document.handle); + const pdf = createPdf(); + const oldText = deferred(); + const firstPage = pdf.pages[0]; + if (!firstPage) { + throw new Error("test document requires a first page"); + } + firstPage.getTextContent.mockReturnValueOnce(oldText.promise); + const firstCancel = vi.fn(); + runtime.renderPdfPage.mockReturnValueOnce({ promise: Promise.resolve(), cancel: firstCancel }); + const { viewer, loaded } = await mount(); + document.loading.resolve(pdf); + await vi.waitFor(() => expect(firstPage.getTextContent).toHaveBeenCalledTimes(1)); + const nextPage = viewer.shadowRoot?.querySelector( + 'button[aria-label="Next page"]', + ); + if (!nextPage || nextPage.disabled) { + throw new Error("loaded document must permit page navigation during rendering"); + } + nextPage.click(); + await ready(viewer, 2); + expect(firstCancel).toHaveBeenCalledTimes(1); + expect(firstPage.cleanup).toHaveBeenCalledTimes(1); + oldText.resolve(pageText("Retired first page")); + await settleLitElement(viewer); + expect(viewer.shadowRoot?.querySelector("canvas")?.getAttribute("data-rendered-page")).toBe( + "2", + ); + expect(viewer.shadowRoot?.querySelector(".page-text")?.textContent).toContain("Page 2 text"); + expect(viewer.shadowRoot?.textContent).not.toContain("Retired first page"); + expect(loaded).toHaveBeenCalledTimes(1); + }); +}); + +describe("document viewer contents lifecycle", () => { + async function mountReady(pageCount = 2) { + const loading = createHandle(); + runtime.openPdfDocument.mockReturnValueOnce(loading.handle); + const result = await mount(); + const pdf = createPdf(pageCount); + loading.loading.resolve(pdf); + await ready(result.viewer); + return { ...result, pdf }; + } + + function button(viewer: HTMLElementTagNameMap["openclaw-document-viewer"], label: string) { + const result = [...(viewer.shadowRoot?.querySelectorAll("button") ?? [])].find( + (candidate) => + candidate.getAttribute("aria-label") === label || candidate.textContent?.trim() === label, + ); + if (!result) { + throw new Error(`Missing document control: ${label}`); + } + return result; + } + + it("opens at page width with minimal footer navigation and copy text inside closed options", async () => { + const { viewer } = await mountReady(); + const root = viewer.shadowRoot; + const canvas = root?.querySelector("canvas.current-page"); + const layout = root?.querySelector(".page-layout"); + const footer = root?.querySelector("footer.toolbar"); + const options = root?.querySelector("details.preview-options"); + if (!canvas || !layout || !footer || !options) { + throw new Error("Missing inline document presentation"); + } + expect(canvas.style.width).toBe("768px"); + expect(Number.parseFloat(canvas.style.height)).toBeGreaterThan(600); + expect(layout.compareDocumentPosition(footer) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect(footer.querySelector(".position")?.textContent?.trim()).toBe("1 / 2"); + expect(footer.querySelector(".position")?.getAttribute("aria-label")).toBe("Page 1 of 2"); + expect(footer.querySelector(".page-unit")?.textContent).toBe("Page"); + expect(root?.querySelector('input[type="number"]')).toBeNull(); + expect(options.open).toBe(false); + expect(options.querySelector(".page-text")?.textContent).toContain("Page 1 text"); + expect(root?.querySelector("form")).toBeNull(); + }); + + it("reflows width-fit pages on width changes without repeated renders from height-only observations", async () => { + const { viewer } = await mountReady(); + const stage = viewer.shadowRoot?.querySelector(".stage"); + const observer = resizeObservers.find((candidate) => candidate.target === stage); + if (!stage || !observer) { + throw new Error("Inline document stage must be observed"); + } + expect(runtime.renderPdfPage).toHaveBeenCalledTimes(1); + Object.defineProperty(stage, "clientHeight", { configurable: true, value: 900 }); + observer.notify(); + await settleLitElement(viewer); + expect(runtime.renderPdfPage).toHaveBeenCalledTimes(1); + Object.defineProperty(stage, "clientWidth", { configurable: true, value: 600 }); + observer.notify(); + observer.notify(); + await ready(viewer); + expect(runtime.renderPdfPage).toHaveBeenCalledTimes(2); + expect( + viewer.shadowRoot?.querySelector("canvas.current-page")?.style.width, + ).toBe("568px"); + observer.notify(); + await settleLitElement(viewer); + expect(runtime.renderPdfPage).toHaveBeenCalledTimes(2); + }); + + it("keeps slide position accessible while PageDown and PageUp navigate the inline preview", async () => { + const { viewer } = await mountReady(); + viewer.pageUnit = "slide"; + await settleLitElement(viewer); + const stage = viewer.shadowRoot?.querySelector(".stage"); + if (!stage) { + throw new Error("Missing keyboard navigation region"); + } + stage.dispatchEvent( + new KeyboardEvent("keydown", { key: "PageDown", bubbles: true, composed: true }), + ); + await ready(viewer, 2); + expect(viewer.shadowRoot?.querySelector(".position")?.textContent?.trim()).toBe("2 / 2"); + expect(viewer.shadowRoot?.querySelector(".position")?.getAttribute("aria-label")).toBe( + "Slide 2 of 2", + ); + expect(button(viewer, "Next slide").disabled).toBe(true); + stage.dispatchEvent( + new KeyboardEvent("keydown", { key: "PageUp", bubbles: true, composed: true }), + ); + await ready(viewer, 1); + expect(button(viewer, "Previous slide").disabled).toBe(true); + }); + + it.each(["ordinary preview", "inline request"])( + "resumes custom zoom when explicitly expanding an %s into a measurable modal stage", + async (mode) => { + const { viewer, pdf, failed } = await mountReady(); + const restoreDialog = installDialogPolyfill(); + try { + if (mode === "inline request") { + viewer.requestChanges = async () => ({ admitted: false }); + await settleLitElement(viewer); + } + button(viewer, "Zoom in").click(); + await settleLitElement(viewer); + await ready(viewer); + if (mode === "inline request") { + button(viewer, "Request changes").click(); + await settleLitElement(viewer); + expect(viewer.shadowRoot?.querySelector("openclaw-modal-dialog")).toBeNull(); + expect(viewer.shadowRoot?.querySelector("form")).toBeNull(); + expect(viewer.shadowRoot?.querySelector(".selection-hint")).not.toBeNull(); + expect( + viewer.shadowRoot?.querySelector(".viewer")?.getAttribute("data-request-mode"), + ).toBe("select"); + } + const customWidth = + viewer.shadowRoot?.querySelector("canvas.current-page")?.style.width; + const renders = runtime.renderPdfPage.mock.calls.length; + const pageReads = pdf.getPage.mock.calls.length; + const pendingPage = deferred(); + pdf.getPage.mockReturnValueOnce(pendingPage.promise); + button(viewer, "Expand document preview").click(); + await vi.waitFor(() => expect(pdf.getPage).toHaveBeenCalledTimes(pageReads + 1)); + const stage = viewer.shadowRoot?.querySelector(".stage"); + const page = pdf.pages[0]; + if (!stage || !page) { + throw new Error("Missing asynchronous modal stage"); + } + stage.style.padding = "16px"; + Object.defineProperties(stage, { + clientWidth: { configurable: true, value: 0 }, + clientHeight: { configurable: true, value: 0 }, + }); + pendingPage.resolve(page); + await settleLitElement(viewer); + expect(viewer.getAttribute("data-render-state")).toBe("loading"); + expect(runtime.renderPdfPage).toHaveBeenCalledTimes(renders); + const observer = resizeObservers.find((candidate) => candidate.target === stage); + if (!observer) { + throw new Error("Modal stage visibility was not observed"); + } + observer.notify(); + await settleLitElement(viewer); + expect(runtime.renderPdfPage).toHaveBeenCalledTimes(renders); + Object.defineProperties(stage, { + clientWidth: { configurable: true, value: 1_000 }, + clientHeight: { configurable: true, value: 700 }, + }); + observer.notify(); + observer.notify(); + await ready(viewer); + expect(runtime.renderPdfPage).toHaveBeenCalledTimes(renders + 1); + expect( + viewer.shadowRoot?.querySelector("canvas.current-page")?.style.width, + ).toBe(customWidth); + observer.notify(); + await settleLitElement(viewer); + expect(runtime.renderPdfPage).toHaveBeenCalledTimes(renders + 1); + expect(failed).not.toHaveBeenCalled(); + button(viewer, "Collapse document preview").click(); + await settleLitElement(viewer); + expect(viewer.shadowRoot?.querySelector("openclaw-modal-dialog")).toBeNull(); + expect(viewer.shadowRoot?.querySelector("form")).toBeNull(); + expect(viewer.shadowRoot?.querySelector(".viewer")?.getAttribute("data-request-mode")).toBe( + "off", + ); + expect(viewer.shadowRoot?.activeElement).toBe( + viewer.shadowRoot?.querySelector(".preview-options > summary"), + ); + } finally { + viewer.remove(); + restoreDialog(); + } + }, + ); + + it("renders only the requested six-page contents window and releases it on close", async () => { + const { viewer, pdf } = await mountReady(20); + expect(runtime.renderPdfPage).toHaveBeenCalledTimes(1); + expect(viewer.shadowRoot?.querySelector("openclaw-document-contents")).toBeNull(); + button(viewer, "Contents").click(); + await settleLitElement(viewer); + const contents = viewer.shadowRoot?.querySelector("openclaw-document-contents"); + if (!contents) { + throw new Error("Missing open contents"); + } + await vi.waitFor(() => + expect(contents.shadowRoot?.querySelectorAll('[data-thumbnail-state="ready"]')).toHaveLength( + 6, + ), + ); + expect(contents.shadowRoot?.querySelectorAll("canvas")).toHaveLength(6); + expect(pdf.getPage.mock.calls.map(([number]) => number)).toEqual([1, 1, 2, 3, 4, 5, 6]); + const oldCanvases = [...(contents.shadowRoot?.querySelectorAll("canvas") ?? [])]; + contents.shadowRoot?.querySelector('[aria-label="Next pages"]')?.click(); + await vi.waitFor(() => { + expect(contents.shadowRoot?.querySelectorAll('[data-thumbnail-state="ready"]')).toHaveLength( + 6, + ); + expect(contents.shadowRoot?.querySelector('canvas[data-page="7"]')).not.toBeNull(); + }); + expect(oldCanvases.every((canvas) => canvas.width === 0 && canvas.height === 0)).toBe(true); + const currentCanvases = [...(contents.shadowRoot?.querySelectorAll("canvas") ?? [])]; + expect(pdf.getPage.mock.calls.slice(7).map(([number]) => number)).toEqual([ + 7, 8, 9, 10, 11, 12, + ]); + button(viewer, "Contents").click(); + await settleLitElement(viewer); + expect(currentCanvases.every((canvas) => canvas.width === 0 && canvas.height === 0)).toBe(true); + }); + + it("cancels a pending thumbnail and stops its closed window from starting more pages", async () => { + const { viewer, pdf } = await mountReady(12); + const rendering = deferred(); + const cancel = vi.fn(); + runtime.renderPdfPage.mockReturnValueOnce({ promise: rendering.promise, cancel }); + button(viewer, "Contents").click(); + await vi.waitFor(() => expect(runtime.renderPdfPage).toHaveBeenCalledTimes(2)); + button(viewer, "Contents").click(); + await settleLitElement(viewer); + expect(cancel).toHaveBeenCalledTimes(1); + rendering.resolve(); + await settleLitElement(viewer); + expect(pdf.getPage).toHaveBeenCalledTimes(2); + expect(viewer.getAttribute("data-render-state")).toBe("ready"); + }); +}); diff --git a/ui/src/components/document-viewer.ts b/ui/src/components/document-viewer.ts new file mode 100644 index 0000000000..ee8edd21bf --- /dev/null +++ b/ui/src/components/document-viewer.ts @@ -0,0 +1,614 @@ +import { html, nothing, type PropertyValues } from "lit"; +import { property, query, state } from "lit/decorators.js"; +import { t } from "../i18n/index.ts"; +import { OpenClawLitElement } from "../lit/openclaw-element.ts"; +import { + DocumentViewerRequestController, + type DocumentChangeHandler, +} from "./document-viewer-request.ts"; +import { createDocumentSelectionTargets } from "./document-viewer-selection.ts"; +import { documentViewerStyles } from "./document-viewer.styles.ts"; +import { toolIcons } from "./icons-tools.ts"; +import type { + PdfDocumentHandle, + PDFDocumentProxy, + PDFPageProxy, + RenderTask, +} from "./pdf-document.runtime.ts"; +import "./modal-dialog.ts"; +import "./document-viewer-contents.ts"; + +const DOCUMENT_MAX_BYTES = 16 * 1024 * 1024; +const PAGE_MAX_PIXELS = 16_000_000; +const PAGE_MAX_DIMENSION = 16_384; +const PAGE_TEXT_MAX_CHARACTERS = 65_536; +const DOCUMENT_OPERATION_TIMEOUT_MS = 30_000; + +type FitMode = "page" | "width" | "custom"; +type RenderState = "loading" | "ready" | "error"; + +class OpenClawDocumentViewer extends OpenClawLitElement { + @property({ attribute: false }) bytes: Uint8Array | ArrayBuffer | undefined; + @property() documentTitle = ""; + @property() pageUnit: "page" | "slide" = "page"; + @property({ attribute: false }) requestChanges: DocumentChangeHandler | undefined; + @query(".stage") private stage?: HTMLDivElement; + @query("canvas.current-page") private canvas?: HTMLCanvasElement; + @state() private renderState: RenderState = "loading"; + @state() private pageNumber = 1; + @state() private pageCount = 0; + @state() private renderedPage = 0; + @state() private pageText = ""; + @state() private fitMode: FitMode = "width"; + @state() private customScale = 1; + @state() private displayScale = 1; + @state() private expanded = false; + @state() private contentsOpen = false; + @state() private failureKind: "password" | "unavailable" = "unavailable"; + + private engine: typeof import("./pdf-document.runtime.ts") | undefined; + private handle: PdfDocumentHandle | undefined; + private pdf: PDFDocumentProxy | undefined; + private pdfPage: PDFPageProxy | undefined; + private renderTask: RenderTask | undefined; + private generation = 0; + private renderGeneration = 0; + private deadline: ReturnType | undefined; + private observer: ResizeObserver | undefined; + private observedStage: HTMLDivElement | undefined; + private lastCanvas: HTMLCanvasElement | undefined; + private waitingForStage = false; + private selectionTargets: ReturnType = []; + private stageWidth = 0; + private stageHeight = 0; + private readonly changeRequest = new DocumentViewerRequestController({ + requestUpdate: () => this.requestUpdate(), + currentPage: () => + this.renderState === "ready" && + this.renderedPage === this.pageNumber && + this.canvas && + this.isConnected + ? { number: this.renderedPage, canvas: this.canvas, targets: this.selectionTargets } + : undefined, + handler: () => this.requestChanges, + }); + + static override styles = documentViewerStyles; + + override connectedCallback(): void { + super.connectedCallback(); + if (this.hasUpdated) { + void this.loadDocument(); + } + } + + override disconnectedCallback(): void { + this.observer?.disconnect(); + this.observedStage = undefined; + this.expanded = false; + void this.retireDocument(); + super.disconnectedCallback(); + } + + protected override updated(changed: PropertyValues): void { + if (changed.has("bytes")) { + void this.loadDocument(); + return; + } + if (this.stage !== this.observedStage) { + this.observer?.disconnect(); + this.observedStage = this.stage; + this.observer ??= new ResizeObserver(() => { + const stage = this.stage; + if ( + stage && + stage.clientWidth > 0 && + stage.clientHeight > 0 && + (this.waitingForStage || + (this.fitMode !== "custom" && stage.clientWidth !== this.stageWidth) || + (this.fitMode === "page" && stage.clientHeight !== this.stageHeight)) + ) { + this.waitingForStage = false; + void this.renderPage(); + } + }); + if (this.stage) { + this.observer.observe(this.stage); + } + } + if ( + changed.has("pageNumber") || + changed.has("fitMode") || + changed.has("customScale") || + changed.has("expanded") + ) { + void this.renderPage(); + } + } + + private clearDeadline(): void { + clearTimeout(this.deadline); + this.deadline = undefined; + } + + private ensureDeadline(): void { + this.deadline ??= setTimeout(() => this.fail("unavailable"), DOCUMENT_OPERATION_TIMEOUT_MS); + } + + private async retireDocument(): Promise { + this.waitingForStage = false; + this.selectionTargets = []; + this.changeRequest.reset(); + this.generation++; + this.renderGeneration++; + this.clearDeadline(); + this.renderTask?.cancel(); + this.renderTask = undefined; + this.pdf = undefined; + this.pdfPage = undefined; + const handle = this.handle; + this.handle = undefined; + if (this.canvas) { + this.canvas.width = this.canvas.height = 0; + } + if (this.lastCanvas && this.lastCanvas !== this.canvas) { + this.lastCanvas.width = this.lastCanvas.height = 0; + } + this.lastCanvas = undefined; + await handle?.destroy(); + } + + private async loadDocument(): Promise { + const retired = this.retireDocument(); + const generation = this.generation; + const bytes = this.bytes; + this.renderState = "loading"; + this.setAttribute("data-render-state", "loading"); + this.pageCount = this.renderedPage = 0; + this.pageNumber = 1; + this.pageText = ""; + this.fitMode = "width"; + this.customScale = 1; + this.contentsOpen = false; + if (!bytes || bytes.byteLength === 0 || bytes.byteLength > DOCUMENT_MAX_BYTES) { + this.fail("unavailable"); + return; + } + this.ensureDeadline(); + try { + await retired; + const engine = await import("./pdf-document.runtime.ts"); + if (!this.isConnected || generation !== this.generation) { + return; + } + this.engine = engine; + const handle = engine.openPdfDocument(bytes, (kind) => { + if (generation === this.generation && this.isConnected) { + this.fail(kind); + } + }); + this.handle = handle; + const pdf = await handle.promise; + if (!this.isConnected || generation !== this.generation) { + return; + } + this.pdf = pdf; + this.pageCount = pdf.numPages; + await this.updateComplete; + await this.renderPage(); + } catch { + if (generation === this.generation && this.isConnected) { + this.fail("unavailable"); + } + } + } + + private fail(kind: "password" | "unavailable"): void { + this.failureKind = kind; + this.renderState = "error"; + this.renderedPage = 0; + this.pageText = ""; + this.setAttribute("data-render-state", "error"); + void this.retireDocument(); + this.dispatchEvent(new CustomEvent("document-viewer-error", { bubbles: true, composed: true })); + } + + private async renderPage(): Promise { + const pdf = this.pdf; + const engine = this.engine; + const canvas = this.canvas; + const stage = this.stage; + if (!pdf || !engine || !canvas || !stage || !this.isConnected) { + return; + } + const generation = this.generation; + const renderGeneration = ++this.renderGeneration; + this.waitingForStage = false; + this.stageWidth = stage.clientWidth; + this.stageHeight = stage.clientHeight; + const current = () => + this.isConnected && + generation === this.generation && + renderGeneration === this.renderGeneration && + canvas === this.canvas; + const previous = this.renderTask; + this.renderTask = undefined; + previous?.cancel(); + this.renderState = "loading"; + this.renderedPage = 0; + this.pageText = ""; + this.selectionTargets = []; + this.setAttribute("data-render-state", "loading"); + this.ensureDeadline(); + try { + await previous?.promise.catch(() => undefined); + if (!current()) { + return; + } + if (this.lastCanvas && this.lastCanvas !== canvas) { + this.lastCanvas.width = this.lastCanvas.height = 0; + } + this.lastCanvas = canvas; + const pageNumber = this.pageNumber; + const page = await pdf.getPage(pageNumber); + if (!current()) { + return; + } + if (this.pdfPage && this.pdfPage !== page) { + this.pdfPage.cleanup(); + } + this.pdfPage = page; + const base = page.getViewport({ scale: 1 }); + const style = getComputedStyle(stage); + const availableWidth = + stage.clientWidth - + Number.parseFloat(style.paddingLeft) - + Number.parseFloat(style.paddingRight); + const availableHeight = + stage.clientHeight - + Number.parseFloat(style.paddingTop) - + Number.parseFloat(style.paddingBottom); + if (availableWidth <= 0 || availableHeight <= 0) { + // A top-layer dialog can mount before its slotted stage is measurable. + // Resume once after visibility changes, including at a custom zoom. + this.waitingForStage = true; + return; + } + const scale = + this.fitMode === "custom" + ? this.customScale + : Math.min( + availableWidth / base.width, + this.fitMode === "page" ? availableHeight / base.height : Infinity, + ); + const width = base.width * scale; + const height = base.height * scale; + if (!Number.isFinite(width + height) || width <= 0 || height <= 0) { + throw new Error("Invalid PDF page dimensions"); + } + const pixelScale = Math.min( + window.devicePixelRatio || 1, + 2, + Math.sqrt(PAGE_MAX_PIXELS / (width * height)), + PAGE_MAX_DIMENSION / width, + PAGE_MAX_DIMENSION / height, + ); + const viewport = page.getViewport({ scale: scale * pixelScale }); + canvas.width = Math.max(1, Math.floor(viewport.width)); + canvas.height = Math.max(1, Math.floor(viewport.height)); + canvas.style.width = `${width}px`; + canvas.style.height = `${height}px`; + const task = engine.renderPdfPage(page, canvas, viewport); + this.renderTask = task; + await task.promise; + if (!current()) { + return; + } + const text = await page.getTextContent(); + if (!current()) { + return; + } + let pageText = ""; + for (const item of text.items) { + if ("str" in item) { + pageText += item.str + (item.hasEOL ? "\n" : " "); + if (pageText.length >= PAGE_TEXT_MAX_CHARACTERS) { + break; + } + } + } + this.pageText = pageText.slice(0, PAGE_TEXT_MAX_CHARACTERS); + this.selectionTargets = createDocumentSelectionTargets(text, base); + this.displayScale = scale; + this.renderedPage = pageNumber; + this.renderState = "ready"; + await this.updateComplete; + if (!current()) { + return; + } + this.setAttribute("data-render-state", "ready"); + this.clearDeadline(); + this.dispatchEvent( + new CustomEvent("document-viewer-load", { bubbles: true, composed: true }), + ); + } catch { + if (current()) { + this.fail("unavailable"); + } + } + } + + private changePage(value: number): void { + if (Number.isFinite(value) && !this.changeRequest.busy) { + this.pageNumber = Math.max(1, Math.min(this.pageCount, Math.floor(value))); + this.changeRequest.changePage(this.pageNumber); + } + } + + private changeZoom(delta: number): void { + this.customScale = Math.min( + 3, + Math.max(0.5, Math.round((this.displayScale + delta) * 100) / 100), + ); + this.fitMode = "custom"; + } + + private collapse = (): void => { + this.changeRequest.reset(); + this.expanded = false; + void this.updateComplete.then(() => { + this.shadowRoot + ?.querySelector(".preview-options > summary") + ?.focus({ preventScroll: true }); + }); + }; + + private handleKeydown = (event: KeyboardEvent): void => { + if (this.expanded && event.key === "Escape") { + event.stopPropagation(); + } + if ( + event.composedPath()[0] instanceof HTMLInputElement || + event.composedPath()[0] instanceof HTMLTextAreaElement || + event.altKey || + event.ctrlKey || + event.metaKey + ) { + return; + } + if (event.key === "PageDown" || event.key === "PageUp") { + event.preventDefault(); + this.changePage(this.pageNumber + (event.key === "PageDown" ? 1 : -1)); + } + }; + + override render() { + const ready = this.pageCount > 0 && this.renderState !== "error"; + const navigationReady = ready && !this.changeRequest.busy; + const slide = this.pageUnit === "slide"; + const position = { current: String(this.pageNumber), total: String(this.pageCount) }; + const content = html`
+ ${this.expanded + ? html`
+ ${this.documentTitle} +
` + : nothing} +
+ ${this.contentsOpen && this.pdf && this.engine + ? html`) => this.changePage(event.detail)} + >` + : nothing} +
+ ${this.renderState === "error" + ? html`` + : nothing} + ${this.renderState === "loading" + ? html`
${t("common.loading")}
` + : nothing} +
+ + ${this.changeRequest.renderOverlay()} +
+ ${this.changeRequest.renderPopover()} +
+
+
+ +
+ + ${this.requestChanges + ? html`` + : nothing} +
+ ⋯ +
+
+ + ${Math.round(this.displayScale * 100)}% + + + + +
+
+ + ${t( + slide + ? "chat.documentViewer.slideTextAlternative" + : "chat.documentViewer.textAlternative", + )} + +
${this.pageText}
+
+ ${slide + ? html`

${t("chat.attachments.powerPointStaticPreview")}

` + : nothing} +
+
+
+
+ ${this.changeRequest.active + ? html`
${t("chat.documentViewer.requestHint")}
` + : nothing} + ${this.changeRequest.renderStatus()} +
`; + return this.expanded + ? html`${content}` + : content; + } +} + +if (!customElements.get("openclaw-document-viewer")) { + customElements.define("openclaw-document-viewer", OpenClawDocumentViewer); +} + +declare global { + interface HTMLElementTagNameMap { + "openclaw-document-viewer": OpenClawDocumentViewer; + } +} diff --git a/ui/src/components/pdf-document.runtime.ts b/ui/src/components/pdf-document.runtime.ts new file mode 100644 index 0000000000..016aeefe13 --- /dev/null +++ b/ui/src/components/pdf-document.runtime.ts @@ -0,0 +1,118 @@ +import { + AnnotationMode, + PDFWorker, + getDocument, + type PDFDocumentProxy, + type PDFPageProxy, + type PageViewport, + type RenderTask, +} from "pdfjs-dist"; +import workerUrl from "pdfjs-dist/build/pdf.worker.min.mjs?worker&url"; +import { pdfAssetUrls } from "virtual:openclaw-pdf-assets"; + +export type { PDFDocumentProxy, PDFPageProxy, RenderTask } from "pdfjs-dist"; + +export type PdfDocumentHandle = { + promise: Promise; + destroy(): Promise; +}; + +const PDF_MAX_IMAGE_PIXELS = 16_000_000; +const WORKER_CLEANUP_TIMEOUT_MS = 1_000; + +function localUrl(value: string): string { + const url = new URL(value, document.baseURI); + if (url.origin !== location.origin || !/^https?:$/u.test(url.protocol)) { + throw new Error("PDF assets must be served by this application"); + } + return url.href; +} + +/** The caller retains its source bytes; PDF.js takes ownership of this private copy. */ +export function openPdfDocument( + bytes: Uint8Array | ArrayBuffer, + onFailure: (kind: "password" | "unavailable") => void, +): PdfDocumentHandle { + const nativeWorker = new Worker(localUrl(workerUrl), { + type: "module", + name: "openclaw-document-preview", + }); + let rejectWorkerFailure: (reason: Error) => void; + const workerFailure = new Promise((_resolve, reject) => { + rejectWorkerFailure = reject; + }); + let destroyed = false; + const fail = (kind: "password" | "unavailable") => { + if (!destroyed) { + rejectWorkerFailure(new Error("PDF worker unavailable")); + onFailure(kind); + } + }; + const onWorkerError = () => fail("unavailable"); + nativeWorker.addEventListener("error", onWorkerError); + // PDF.js 6.4 display/api.js initializes a supplied port directly. Supplying + // our native Worker prevents its automatic main-thread fake-worker fallback. + let worker: PDFWorker | undefined; + let task: ReturnType; + try { + worker = PDFWorker.create({ port: nativeWorker }); + task = getDocument({ + data: bytes instanceof Uint8Array ? bytes.slice() : bytes.slice(0), + worker, + cMapUrl: localUrl(pdfAssetUrls.cMapUrl), + cMapPacked: true, + standardFontDataUrl: localUrl(pdfAssetUrls.standardFontDataUrl), + wasmUrl: localUrl(pdfAssetUrls.wasmUrl), + iccUrl: localUrl(pdfAssetUrls.iccUrl), + useWorkerFetch: true, + disableFontFace: true, + useSystemFonts: false, + enableXfa: false, + // PDF.js otherwise reports success after silently omitting oversized scans. + stopAtErrors: true, + maxImageSize: PDF_MAX_IMAGE_PIXELS, + canvasMaxAreaInBytes: PDF_MAX_IMAGE_PIXELS * 4, + }); + } catch (error) { + nativeWorker.removeEventListener("error", onWorkerError); + worker?.destroy(); + nativeWorker.terminate(); + throw error; + } + task.onPassword = () => fail("password"); + return { + promise: Promise.race([task.promise, workerFailure]), + async destroy() { + if (destroyed) { + return; + } + destroyed = true; + rejectWorkerFailure(new Error("PDF preview retired")); + nativeWorker.removeEventListener("error", onWorkerError); + let deadline: ReturnType | undefined; + try { + // A failed bootstrap may leave PDF.js's setup promise pending. Always + // terminate our own port even when graceful task destruction cannot finish. + await Promise.race([ + task.destroy().catch(() => undefined), + new Promise((resolve) => { + deadline = setTimeout(resolve, WORKER_CLEANUP_TIMEOUT_MS); + }), + ]); + } finally { + clearTimeout(deadline); + worker?.destroy(); + nativeWorker.terminate(); + } + }, + }; +} + +/** Preserve static annotation appearances without installing interactive or scripting layers. */ +export function renderPdfPage( + page: PDFPageProxy, + canvas: HTMLCanvasElement, + viewport: PageViewport, +): RenderTask { + return page.render({ canvas, viewport, annotationMode: AnnotationMode.ENABLE }); +} diff --git a/ui/src/e2e/chat-attachment-read-lifecycle.e2e.test.ts b/ui/src/e2e/chat-attachment-read-lifecycle.e2e.test.ts index 0d0f6394aa..6d18611349 100644 --- a/ui/src/e2e/chat-attachment-read-lifecycle.e2e.test.ts +++ b/ui/src/e2e/chat-attachment-read-lifecycle.e2e.test.ts @@ -1,25 +1,19 @@ -import { mkdir, mkdtemp, rm } from "node:fs/promises"; -import { createServer } from "node:http"; -import { tmpdir } from "node:os"; +import { mkdir } from "node:fs/promises"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { chromium, type Browser, type Locator, type Page } from "playwright"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { PLATFORMCLAW_WEB_GATEWAY_METHODS } from "../../../packages/platformclaw-control-plane/src/browser-gateway-policy.ts"; -import { - createPlatformClawWebAssetHandler, - PLATFORMCLAW_WEB_TERMINAL_ENABLED, -} from "../../../packages/platformclaw-control-plane/src/web-assets.ts"; import { canRunPlaywrightChromium, - buildProductionControlUiE2e, controlUiSessionUrl, - installMockGateway as installControlUiMockGateway, resolvePlaywrightChromiumExecutablePath, type ControlUiE2eServer, type ControlUiMockGatewayScenario, } from "../test-helpers/control-ui-e2e.ts"; +import { + installPlatformClawMockGateway as installMockGateway, + startPlatformClawControlUiE2eServer, +} from "../test-helpers/platformclaw-control-ui-e2e.ts"; const chromiumExecutablePath = resolvePlaywrightChromiumExecutablePath(chromium.executablePath()); const chromiumAvailable = canRunPlaywrightChromium(chromiumExecutablePath); @@ -32,15 +26,6 @@ const DROPPED_PNG_B64 = const captureUiProof = process.env.OPENCLAW_CAPTURE_UI_PROOF === "1"; const proofDir = path.resolve(".artifacts/control-ui-e2e/clipboard-image-preview-csp"); -function installMockGateway(page: Page, scenario: ControlUiMockGatewayScenario = {}) { - return installControlUiMockGateway(page, { - basePath: "/platformclaw/app", - featureMethods: [...PLATFORMCLAW_WEB_GATEWAY_METHODS], - terminalEnabled: PLATFORMCLAW_WEB_TERMINAL_ENABLED, - ...scenario, - }); -} - async function newPage() { const viewport = { height: 900, width: 1280 }; const context = await browser.newContext({ @@ -50,23 +35,7 @@ async function newPage() { viewport, }); const page = await context.newPage(); - await page.route("**/platformclaw/api/auth/session", (route) => - route.fulfill({ - json: { - authenticated: true, - user: { - accountId: "preview.one", - displayName: "Preview One", - department: "Platform Lab", - globalRole: "member", - }, - agent: { agentId: "main", state: "active" }, - session: { idleExpiresAt: Date.now() + 60_000, absoluteExpiresAt: Date.now() + 120_000 }, - }, - }), - ); await page.addInitScript(() => { - localStorage.setItem("platformclaw.product-tour.v1.completed", "true"); const violations: string[] = []; (globalThis as unknown as { imageCspViolations: string[] }).imageCspViolations = violations; document.addEventListener("securitypolicyviolation", (event) => { @@ -148,69 +117,22 @@ async function pastePng(composer: Locator): Promise { let browser: Browser; let server: ControlUiE2eServer; -let buildDirectory: string | undefined; describeControlUiE2e("Control UI chat attachment read lifecycle", () => { beforeAll(async () => { if (!chromiumAvailable) { throw new Error(`Playwright Chromium is unavailable at ${chromiumExecutablePath}`); } - buildDirectory = await mkdtemp(path.join(tmpdir(), "platformclaw-employee-preview-")); - await buildProductionControlUiE2e(buildDirectory, "employee-image-preview"); - const [{ build }, { default: loginConfig }] = await Promise.all([ - import("vite"), - import("../../vite.platformclaw-login.config.ts"), - ]); - await build({ - ...loginConfig({ outDir: buildDirectory }), - configFile: false, - logLevel: "error", - root: fileURLToPath(new URL("../..", import.meta.url)), - }); + server = await startPlatformClawControlUiE2eServer("employee-image-preview"); if (captureUiProof) { await mkdir(proofDir, { recursive: true }); } - // Serve the production bundle through the employee document owner. Vite's - // preview server and the regular Gateway use different CSP policies. - const httpServer = createServer((req, res) => { - void (async () => { - if (!(await assets.handlePublic(req, res)) && !(await assets.handleApplication(req, res))) { - res.writeHead(404).end(); - } - })().catch(() => res.writeHead(500).end()); - }); - await new Promise((resolve, reject) => { - httpServer.once("error", reject); - httpServer.listen(0, "127.0.0.1", resolve); - }); - const address = httpServer.address(); - if (!address || typeof address === "string") { - throw new Error("Employee web server did not expose a TCP port"); - } - const origin = `http://127.0.0.1:${address.port}`; - server = { - baseUrl: `${origin}/platformclaw/app/`, - close: () => - new Promise((resolve, reject) => { - httpServer.close((error) => (error ? reject(error) : resolve())); - }), - }; - let assets: ReturnType; - try { - assets = createPlatformClawWebAssetHandler(buildDirectory, { publicOrigin: origin }); - } catch (error) { - await server.close(); - throw error; - } browser = await chromium.launch({ executablePath: chromiumExecutablePath }); }); afterAll(async () => { await browser?.close(); await server?.close(); - if (buildDirectory) { - await rm(buildDirectory, { recursive: true, force: true }); - } }); it("waits for a pasted image before sending its complete gateway payload", async () => { diff --git a/ui/src/e2e/document-viewer-rendering.e2e.test.ts b/ui/src/e2e/document-viewer-rendering.e2e.test.ts new file mode 100644 index 0000000000..d09a0505ff --- /dev/null +++ b/ui/src/e2e/document-viewer-rendering.e2e.test.ts @@ -0,0 +1,400 @@ +import { mkdir, readFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { chromium, type Browser } from "playwright"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + resolvePlaywrightChromiumExecutablePath, + type ControlUiE2eServer, +} from "../test-helpers/control-ui-e2e.ts"; +import { + installPlatformClawMockGateway, + observePlatformClawPreviewCsp, + startPlatformClawControlUiE2eServer, +} from "../test-helpers/platformclaw-control-ui-e2e.ts"; +import { createScanPreviewPdf } from "./platformclaw-document-preview.test-fixtures.ts"; + +const proofDir = path.resolve( + ".artifacts/control-ui-e2e/platformclaw-canvas/document-previews/rendering-boundaries", +); + +describe("document viewer real PDF rendering failures", () => { + let browser: Browser; + let server: ControlUiE2eServer; + + beforeAll(async () => { + await mkdir(proofDir, { recursive: true }); + server = await startPlatformClawControlUiE2eServer("document-rendering-boundaries"); + browser = await chromium.launch({ + executablePath: resolvePlaywrightChromiumExecutablePath(chromium.executablePath()), + }); + }); + + afterAll(async () => { + await browser?.close(); + await server?.close(); + }); + + it("renders an admissible scan and visibly rejects a compressed oversized scan", async () => { + const sessionKey = "agent:document-rendering:main"; + const fixtures = [ + { filename: "readable-scan.pdf", bytes: createScanPreviewPdf(250, 350) }, + { filename: "oversized-scan.pdf", bytes: createScanPreviewPdf(5_000, 7_000) }, + ].map((fixture, index) => ({ + filename: fixture.filename, + bytes: fixture.bytes, + artifactId: `artifact_managed_media_11111111-1111-4111-8111-${String(index + 1).padStart(12, "0")}`, + source: `/api/chat/media/outgoing/${encodeURIComponent(sessionKey)}/scan-${index + 1}/full`, + })); + const readable = fixtures[0]!; + const oversized = fixtures[1]!; + expect(oversized.bytes.length).toBeLessThan(16 * 1024 * 1024); + const context = await browser.newContext({ + acceptDownloads: true, + locale: "en-US", + serviceWorkers: "block", + viewport: { width: 1280, height: 900 }, + ...(process.env.OPENCLAW_CAPTURE_UI_PROOF === "1" + ? { recordVideo: { dir: proofDir, size: { width: 1280, height: 900 } } } + : {}), + }); + try { + // Only identity/history and attachment transport are synthetic. This test + // exercises the real production CSP, worker and card/renderer failure path; + // Basic/VM snapshot ownership is covered by the separate runtime scenario. + await context.route("**/api/chat/media/outgoing/**", async (route) => { + const url = new URL(route.request().url()); + const fixture = fixtures.find((candidate) => candidate.source === url.pathname); + if (!fixture) { + throw new Error("unexpected scan fixture request"); + } + expect(url.searchParams.get("preview")).toBeNull(); + await route.fulfill({ + status: 200, + body: fixture.bytes, + contentType: "application/pdf", + headers: { "Content-Disposition": `attachment; filename="${fixture.filename}"` }, + }); + }); + const page = await context.newPage(); + const mediaResponses = new Map }>(); + page.on("response", (response) => { + const pathname = new URL(response.url()).pathname; + if ( + !response.request().isNavigationRequest() && + fixtures.some((fixture) => fixture.source === pathname) + ) { + mediaResponses.set(pathname, { status: response.status(), body: response.body() }); + } + }); + const cspViolations = await observePlatformClawPreviewCsp(page); + const gateway = await installPlatformClawMockGateway(page, { + assistantAgentId: "document-rendering", + defaultAgentId: "document-rendering", + sessionKey, + historyMessages: [ + { + role: "assistant", + content: fixtures.map((fixture) => ({ + type: "attachment", + attachment: { + kind: "document", + label: fixture.filename, + mimeType: "application/pdf", + sizeBytes: fixture.bytes.length, + artifactId: fixture.artifactId, + url: fixture.source, + }, + })), + timestamp: 1, + }, + ], + methodResponses: { + "artifacts.download": { + cases: fixtures.map((fixture) => ({ + match: { sessionKey, artifactId: fixture.artifactId }, + response: { + artifact: { + id: fixture.artifactId, + title: fixture.filename, + type: "file", + mimeType: "application/pdf", + download: { mode: "url" }, + }, + url: `${fixture.source}?mediaTicket=rendering-fixture`, + expiresAt: new Date(Date.now() + 120_000).toISOString(), + }, + })), + }, + }, + }); + const documentResponse = await page.goto(`${server.baseUrl}chat`); + expect(documentResponse?.headers()["content-security-policy"]).toContain( + "default-src 'self'", + ); + await gateway.waitForRequest("chat.startup"); + + // A successful scan with the same image encoding rules out a broken worker, + // font/asset setup or universal image rejection masquerading as the fix. + const readableCard = page + .locator("openclaw-chat-document-attachment") + .filter({ hasText: readable.filename }); + await readableCard.scrollIntoViewIfNeeded(); + await readableCard.getByRole("button", { name: "Close preview", exact: true }).waitFor(); + const readableViewer = readableCard.locator("openclaw-document-viewer"); + await expect.poll(() => readableViewer.getAttribute("data-render-state")).toBe("ready"); + const scanPainted = await readableViewer + .locator("canvas.current-page") + .evaluate((element) => { + if (!(element instanceof HTMLCanvasElement)) { + throw new Error("readable scan has no canvas"); + } + const pixel = element + .getContext("2d") + ?.getImageData( + Math.floor(element.width / 2), + Math.floor(element.height / 2), + 1, + 1, + ).data; + if (!pixel) { + throw new Error("readable scan has no pixel context"); + } + const [red = 0, green = 0, blue = 0, alpha = 0] = pixel; + return ( + alpha === 255 && + red > 100 && + red < 170 && + Math.abs(red - green) < 3 && + Math.abs(red - blue) < 3 + ); + }); + expect(scanPainted).toBe(true); + await readableCard.getByRole("button", { name: "Close preview", exact: true }).click(); + + const oversizedCard = page + .locator("openclaw-chat-document-attachment") + .filter({ hasText: oversized.filename }); + await oversizedCard.scrollIntoViewIfNeeded(); + await oversizedCard.getByRole("button", { name: "Close preview", exact: true }).waitFor(); + await expect.poll(() => mediaResponses.has(oversized.source)).toBe(true); + const response = mediaResponses.get(oversized.source); + if (!response) { + throw new Error("oversized scan did not load automatically"); + } + expect(response.status).toBe(200); + expect(await response.body).toEqual(oversized.bytes); + const oversizedViewer = oversizedCard.locator("openclaw-document-viewer"); + await expect.poll(() => oversizedViewer.getAttribute("data-render-state")).toBe("error"); + const alert = oversizedCard.getByRole("alert"); + await alert.waitFor({ state: "visible" }); + expect(await alert.textContent()).toContain("download the original file"); + expect(await oversizedViewer.locator("canvas.current-page").isVisible()).toBe(false); + expect(cspViolations).toEqual([]); + await page.screenshot({ + path: path.join(proofDir, "oversized-scan-visible-error.png"), + fullPage: true, + }); + + const [download] = await Promise.all([ + page.waitForEvent("download"), + oversizedCard.getByRole("link", { name: "Download file", exact: true }).click(), + ]); + expect(download.suggestedFilename()).toBe(oversized.filename); + const saved = await download.path(); + if (!saved) { + throw new Error("oversized scan original download did not complete"); + } + expect(await readFile(saved)).toEqual(oversized.bytes); + expect(await gateway.getRequests("chat.send")).toHaveLength(0); + await oversizedCard.getByRole("button", { name: "Close preview", exact: true }).click(); + await oversizedCard.getByRole("button", { name: "Preview", exact: true }).waitFor(); + expect(await oversizedCard.locator("openclaw-document-viewer").count()).toBe(0); + } finally { + await context.close(); + } + }); + + it("rejects public operator-stream failures and recovers an ordinary cancelled render", async () => { + const displayPath = createRequire(import.meta.url).resolve("pdfjs-dist"); + const [displaySource, workerSource] = await Promise.all([ + readFile(displayPath), + readFile(path.join(path.dirname(displayPath), "pdf.worker.mjs")), + ]); + const context = await browser.newContext({ + locale: "en-US", + serviceWorkers: "block", + }); + try { + // These same-origin test assets are the installed package bytes, including + // its approved patch. No fake worker, RPC observer or renderer mock is used. + await context.route("**/__pdfjs_public_api/**", async (route) => { + const filename = path.posix.basename(new URL(route.request().url()).pathname); + if (filename !== "pdf.mjs" && filename !== "pdf.worker.mjs") { + throw new Error("unexpected PDF public API test asset"); + } + await route.fulfill({ + contentType: "text/javascript", + body: filename === "pdf.mjs" ? displaySource : workerSource, + }); + }); + const page = await context.newPage(); + const cspViolations = await observePlatformClawPreviewCsp(page); + await installPlatformClawMockGateway(page); + const response = await page.goto(`${server.baseUrl}chat`); + expect(response?.headers()["content-security-policy"]).toContain("default-src 'self'"); + const origin = new URL(page.url()).origin; + await page.addScriptTag({ + type: "module", + url: `${origin}/__pdfjs_public_api/pdf.mjs`, + }); + const result = await page.evaluate( + async ({ workerUrl, readable, oversized }) => { + // The shipped module deliberately publishes this public namespace. + // Loading its external module URL avoids Vitest rewriting a dynamic + // import inside the serialized browser function into an SSR helper. + const browserGlobal: typeof globalThis & { + pdfjsLib?: Pick; + } = globalThis; + const engine = browserGlobal.pdfjsLib; + if (!engine) { + throw new Error("PDF.js public module did not initialize"); + } + type Outcome = + | { state: "resolved" } + | { state: "rejected"; name: string; message: string }; + const outcome = (promise: Promise): Promise => + promise.then( + () => ({ state: "resolved" }), + (reason: unknown) => ({ + state: "rejected", + name: reason instanceof Error ? reason.name : "Unknown", + message: reason instanceof Error ? reason.message : String(reason), + }), + ); + const withPdf = async ( + bytes: number[], + exercise: (pdf: import("pdfjs-dist").PDFDocumentProxy) => Promise, + ): Promise => { + const nativeWorker = new Worker(workerUrl, { type: "module" }); + let worker: import("pdfjs-dist").PDFWorker | undefined; + let loading: import("pdfjs-dist").PDFDocumentLoadingTask | undefined; + try { + worker = engine.PDFWorker.create({ port: nativeWorker }); + loading = engine.getDocument({ + data: Uint8Array.from(bytes), + worker, + stopAtErrors: true, + maxImageSize: 16_000_000, + disableFontFace: true, + useSystemFonts: false, + enableXfa: false, + }); + return await exercise(await loading.promise); + } finally { + try { + await loading?.destroy(); + } finally { + worker?.destroy(); + nativeWorker.terminate(); + } + } + }; + const canvasFor = (viewport: { width: number; height: number }) => { + const canvas = document.createElement("canvas"); + canvas.width = Math.ceil(viewport.width); + canvas.height = Math.ceil(viewport.height); + return canvas; + }; + const failures = await withPdf(oversized, async (pdf) => { + const pdfPage = await pdf.getPage(1); + const viewport = pdfPage.getViewport({ scale: 1 }); + const firstCanvas = canvasFor(viewport); + const secondCanvas = canvasFor(viewport); + const first = pdfPage.render({ canvas: firstCanvas, viewport }); + const parallel = pdfPage.render({ canvas: secondCanvas, viewport }); + const [firstRender, parallelRender] = await Promise.all([ + outcome(first.promise), + outcome(parallel.promise), + ]); + const replayRender = await outcome( + pdfPage.render({ canvas: firstCanvas, viewport }).promise, + ); + // getOperatorList has its own public intent/cache. Exercise both its + // first failed read and the cached rejected promise independently. + const operatorList = await outcome(pdfPage.getOperatorList()); + const operatorListReplay = await outcome(pdfPage.getOperatorList()); + return { + outcomes: { + firstRender, + parallelRender, + replayRender, + operatorList, + operatorListReplay, + }, + cleanup: pdfPage.cleanup(), + }; + }); + const recovery = await withPdf(readable, async (pdf) => { + const pdfPage = await pdf.getPage(1); + const viewport = pdfPage.getViewport({ scale: 1 }); + const canvas = canvasFor(viewport); + const first = pdfPage.render({ canvas, viewport }); + let cancelledFromContinue = false; + first.onContinue = () => { + cancelledFromContinue = true; + first.cancel(); + }; + const cancellation = await outcome(first.promise); + // Reuse the same page and canvas immediately, including the normal + // delayed stream-cancellation window, without a cleanup/reset shortcut. + await pdfPage.render({ canvas, viewport }).promise; + const pixel = canvas + .getContext("2d") + ?.getImageData( + Math.floor(canvas.width / 2), + Math.floor(canvas.height / 2), + 1, + 1, + ).data; + const red = pixel?.[0] ?? 0; + return { + cancelledFromContinue, + cancellation, + painted: + pixel?.[3] === 255 && + red > 100 && + red < 170 && + Math.abs(red - (pixel?.[1] ?? 0)) < 3 && + Math.abs(red - (pixel?.[2] ?? 0)) < 3, + cleanup: pdfPage.cleanup(), + }; + }); + return { version: engine.version, failures, recovery }; + }, + { + workerUrl: `${origin}/__pdfjs_public_api/pdf.worker.mjs`, + readable: Array.from(createScanPreviewPdf(250, 350)), + oversized: Array.from(createScanPreviewPdf(5_000, 7_000)), + }, + ); + expect(result.version).toBe("6.4.299"); + for (const failure of Object.values(result.failures.outcomes)) { + expect(failure).toMatchObject({ state: "rejected" }); + if (failure.state === "rejected") { + expect(failure.message).toContain("Image exceeded maximum allowed size"); + } + } + expect(result.failures.cleanup).toBe(true); + expect(result.recovery).toMatchObject({ + cancelledFromContinue: true, + cancellation: { state: "rejected", name: "RenderingCancelledException" }, + painted: true, + cleanup: true, + }); + expect(cspViolations).toEqual([]); + } finally { + await context.close(); + } + }); +}); diff --git a/ui/src/e2e/platformclaw-document-change-browser.test-support.ts b/ui/src/e2e/platformclaw-document-change-browser.test-support.ts new file mode 100644 index 0000000000..01a6970bf6 --- /dev/null +++ b/ui/src/e2e/platformclaw-document-change-browser.test-support.ts @@ -0,0 +1,393 @@ +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import type { Locator, Page } from "playwright"; +import { expect } from "vitest"; +import type { MockGatewayControls } from "../test-helpers/control-ui-e2e.ts"; +import { + clickDocumentPreviewOption, + expectFittedPage, + expectRenderedPage, +} from "./platformclaw-document-preview-browser.test-support.ts"; +import type { DocumentPreviewFixture } from "./platformclaw-document-preview.test-fixtures.ts"; + +async function scrollDocumentToTop(viewer: Locator): Promise { + const stage = viewer.locator(".stage"); + await stage.scrollIntoViewIfNeeded(); + await stage.evaluate((element) => { + element.scrollTop = 0; + element.scrollLeft = 0; + }); +} + +async function dragDocumentRegion(page: Page, viewer: Locator): Promise { + await scrollDocumentToTop(viewer); + const bounds = await viewer.locator("canvas.current-page").boundingBox(); + if (!bounds || bounds.width <= 0 || bounds.height <= 0) { + throw new Error("selection requires a visible rendered document"); + } + await page.mouse.move(bounds.x + bounds.width * 0.15, bounds.y + bounds.height * 0.15); + await page.mouse.down(); + await page.mouse.move(bounds.x + bounds.width * 0.65, bounds.y + bounds.height * 0.55, { + steps: 5, + }); + await page.mouse.up(); +} + +async function clickDocumentTitle( + page: Page, + viewer: Locator, + presentation: boolean, +): Promise { + await scrollDocumentToTop(viewer); + const point = await viewer.locator("canvas.current-page").evaluate((element, whiteInk) => { + if (!(element instanceof HTMLCanvasElement)) { + throw new Error("document title has no canvas"); + } + const context = element.getContext("2d"); + if (!context) { + throw new Error("document title has no pixels"); + } + const pixels = context.getImageData(0, 0, element.width, Math.ceil(element.height * 0.3)); + // Independent fixture paint: PDF title ink is teal; PPT title ink is white + // on teal. Hit the actual rendered glyph, not private computed text boxes. + for (let y = Math.floor(element.height * 0.03); y < pixels.height; y++) { + for (let x = Math.floor(element.width * 0.04); x < element.width * 0.9; x++) { + const offset = (y * element.width + x) * 4; + const red = pixels.data[offset] ?? 0; + const green = pixels.data[offset + 1] ?? 0; + const blue = pixels.data[offset + 2] ?? 0; + const alpha = pixels.data[offset + 3] ?? 0; + const ink = whiteInk + ? red > 240 && green > 240 && blue > 240 + : red < 70 && green > 90 && blue > 80 && green - red > 70; + if (alpha > 240 && ink) { + const bounds = element.getBoundingClientRect(); + return { + x: bounds.x + ((x + 0.5) * bounds.width) / element.width, + y: bounds.y + ((y + 0.5) * bounds.height) / element.height, + }; + } + } + } + throw new Error("rendered title ink was not found"); + }, presentation); + await page.mouse.click(point.x, point.y); +} + +async function expectPopoverInsideStage(viewer: Locator): Promise { + await expect + .poll(() => + viewer.locator(".request-popover").evaluate((element) => { + const stage = element.closest(".stage"); + if (!(stage instanceof HTMLElement)) { + throw new Error("request popover lost its inline stage"); + } + const popup = element.getBoundingClientRect(); + const bounds = stage.getBoundingClientRect(); + return ( + getComputedStyle(element).visibility !== "hidden" && + popup.width > 0 && + popup.height > 0 && + popup.left >= bounds.left - 1 && + popup.right <= bounds.right + 1 && + popup.top >= bounds.top - 1 && + popup.bottom <= bounds.bottom + 1 + ); + }), + ) + .toBe(true); +} + +/** Run only after the snapshot/download/foreign-owner proof has made no chat requests. */ +export async function exerciseDocumentChangeRequest(params: { + page: Page; + fixture: DocumentPreviewFixture; + target: string; + proofDir: string; + sessionKey: string; + gateway: Pick; + source: { messageId: string; artifactId: string }; +}): Promise { + const { page, fixture, target, proofDir, sessionKey, gateway, source } = params; + const originalViewport = page.viewportSize(); + if (!originalViewport) { + throw new Error("document request proof requires a fixed viewport"); + } + // A legitimate tall viewport retains the inline preview and surrounding chat. + const desktopViewport = { + width: originalViewport.width, + height: Math.max(originalViewport.height, 1200), + }; + await page.setViewportSize(desktopViewport); + const card = page + .locator("openclaw-chat-document-attachment") + .filter({ hasText: fixture.filename }); + const viewer = card.locator("openclaw-document-viewer"); + const pageUnit = fixture.format === "powerpoint" ? "slide" : "page"; + const total = fixture.format === "powerpoint" ? 2 : 1; + const toggle = viewer.getByRole("button", { name: "Request changes", exact: true }); + const popover = viewer.locator(".request-popover"); + const instructionInput = viewer.getByRole("textbox", { + name: "Describe the change", + exact: true, + }); + const expectInline = async () => { + expect(await viewer.locator("openclaw-modal-dialog").count()).toBe(0); + expect(await viewer.locator('summary[aria-label="Preview options"]').isVisible()).toBe(true); + expect( + await viewer + .locator("details.preview-options") + .evaluate((element) => element instanceof HTMLDetailsElement && !element.open), + ).toBe(true); + }; + const open = async () => { + await card.getByRole("button", { name: "Preview", exact: true }).click(); + await expectRenderedPage(viewer, 1, total, fixture.format === "pdf", pageUnit); + await expectFittedPage(viewer, "width"); + await expectInline(); + }; + const expectModeOff = async () => { + expect(await viewer.locator(".viewer").getAttribute("data-request-mode")).toBe("off"); + expect(await popover.count()).toBe(0); + expect(await viewer.locator(".selection-region").count()).toBe(0); + expect(await gateway.getRequests("chat.send")).toHaveLength(0); + }; + const begin = async () => { + await toggle.click(); + await expect + .poll(() => viewer.locator(".viewer").getAttribute("data-request-mode")) + .toBe("select"); + await expectRenderedPage(viewer, 1, total, fixture.format === "pdf", pageUnit); + await expectFittedPage(viewer, "width"); + await expectInline(); + expect(await toggle.getAttribute("aria-pressed")).toBe("true"); + expect(await popover.count()).toBe(0); + }; + const selectRegion = async () => { + await begin(); + await dragDocumentRegion(page, viewer); + await viewer.locator(".selection-region").waitFor({ state: "visible" }); + await popover.waitFor({ state: "visible" }); + await expectPopoverInsideStage(viewer); + expect(await viewer.locator(".selected-text").count()).toBe(0); + expect(await instructionInput.inputValue()).toBe(""); + }; + await open(); + await clickDocumentTitle(page, viewer, fixture.format === "powerpoint"); + await dragDocumentRegion(page, viewer); + await expectModeOff(); + + await begin(); + await clickDocumentTitle(page, viewer, fixture.format === "powerpoint"); + await popover.waitFor({ state: "visible" }); + const selectedText = viewer.locator(".selected-text"); + await selectedText.waitFor({ state: "visible" }); + expect(await selectedText.textContent()).toContain("Owned"); + await expectPopoverInsideStage(viewer); + expect(await gateway.getRequests("chat.send")).toHaveLength(0); + await instructionInput.fill("제목을 ‘2026 실행 계획’으로 바꿔줘"); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-request-text-selection.png`), + fullPage: true, + }); + await instructionInput.fill("Discard this clicked title draft"); + await toggle.click(); + await popover.waitFor({ state: "detached" }); + await expectModeOff(); + await expectInline(); + + await selectRegion(); + await instructionInput.fill("Discard this cancelled draft"); + await viewer.getByRole("button", { name: "Cancel request", exact: true }).click(); + await popover.waitFor({ state: "detached" }); + expect(await viewer.locator(".viewer").getAttribute("data-request-mode")).toBe("select"); + expect(await gateway.getRequests("chat.send")).toHaveLength(0); + await toggle.click(); + await expectModeOff(); + + await selectRegion(); + await instructionInput.fill("Discard this closed preview draft"); + await clickDocumentPreviewOption(viewer, "Expand document preview"); + await viewer.getByRole("dialog").waitFor({ state: "visible" }); + await expectRenderedPage(viewer, 1, total, fixture.format === "pdf", pageUnit); + await expectFittedPage(viewer, "width"); + await expectPopoverInsideStage(viewer); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-request-expanded.png`), + fullPage: true, + }); + await clickDocumentPreviewOption(viewer, "Collapse document preview"); + await viewer.locator("openclaw-modal-dialog").waitFor({ state: "detached" }); + await expectRenderedPage(viewer, 1, total, fixture.format === "pdf", pageUnit); + await expectModeOff(); + await expectInline(); + await card.getByRole("button", { name: "Close preview", exact: true }).click(); + await viewer.waitFor({ state: "detached" }); + await open(); + await expectModeOff(); + + const instruction = "Change the selected teal region to blue and keep the surrounding text."; + await selectRegion(); + await instructionInput.fill(instruction); + await expectInline(); + expect(await gateway.getRequests("chat.send")).toHaveLength(0); + await card.scrollIntoViewIfNeeded(); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-request-selection.png`), + fullPage: true, + }); + await page.setViewportSize({ width: 390, height: 844 }); + await expectRenderedPage(viewer, 1, total, fixture.format === "pdf", pageUnit); + await expectFittedPage(viewer, "width"); + await expectInline(); + await expectPopoverInsideStage(viewer); + await viewer.locator(".stage").scrollIntoViewIfNeeded(); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-request-mobile-document.png`), + fullPage: true, + }); + await popover.scrollIntoViewIfNeeded(); + const mobileBox = await viewer.boundingBox(); + if (!mobileBox) { + throw new Error("mobile inline preview disappeared"); + } + expect(mobileBox.x).toBeGreaterThanOrEqual(0); + expect(mobileBox.x + mobileBox.width).toBeLessThanOrEqual(390); + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(391); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-request-mobile-form.png`), + fullPage: true, + }); + await page.setViewportSize(desktopViewport); + await expectRenderedPage(viewer, 1, total, fixture.format === "pdf", pageUnit); + await expectFittedPage(viewer, "width"); + await expectInline(); + await expectPopoverInsideStage(viewer); + const canvasSize = await viewer.locator("canvas.current-page").evaluate((element) => { + if (!(element instanceof HTMLCanvasElement)) { + throw new Error("document selection lost its rendered canvas"); + } + return { width: element.width, height: element.height }; + }); + expect(await gateway.getRequests("chat.send")).toHaveLength(0); + await viewer.getByRole("button", { name: "Send request", exact: true }).click(); + const request = await gateway.waitForRequest("chat.send"); + const sent = request.params; + if (!isRecord(sent) || typeof sent.message !== "string") { + throw new Error("document request did not use the canonical chat payload"); + } + expect(sent).toMatchObject({ sessionKey, replyToId: source.messageId, deliver: false }); + const pageLabel = `${pageUnit === "slide" ? "Slide" : "Page"} 1`; + expect(sent.message).toContain(`Requested changes to ${fixture.filename} — ${pageLabel}`); + expect(sent.message).toContain(instruction); + expect(sent.message).not.toContain("Selected text:"); + expect(sent.message).toContain("The attached image shows the selected region."); + expect(sent.message).toContain( + "The original document is referenced in the reply, not reattached.", + ); + expect(sent.message).not.toMatch( + /Source:|sourceReference|previewRevision|messageId|artifactId|mediaTicket|https?:\/\/|data:image\/|\/api\/chat\/media\//u, + ); + expect(sent.message).not.toContain(source.messageId); + expect(sent.message).not.toContain(source.artifactId); + const region = + /^Selected region: left ([\d.]+)%, top ([\d.]+)%, width ([\d.]+)%, height ([\d.]+)%\.$/mu.exec( + sent.message, + ); + if (!region) { + throw new Error("document request has no human-readable region context"); + } + expect(Number(region[1])).toBeCloseTo(15, 0); + expect(Number(region[2])).toBeCloseTo(15, 0); + expect(Number(region[3])).toBeCloseTo(50, 0); + expect(Number(region[4])).toBeCloseTo(40, 0); + expect(sent.attachments).toHaveLength(1); + if (!Array.isArray(sent.attachments) || !isRecord(sent.attachments[0])) { + throw new Error("document request has no crop attachment"); + } + const attachment = sent.attachments[0]; + const cropContent = attachment.content; + if (typeof cropContent !== "string") { + throw new Error("document request crop has no encoded image bytes"); + } + expect(attachment).toMatchObject({ + type: "image", + mimeType: "image/png", + fileName: `document-selection-${pageUnit}-1.png`, + }); + const cropBytes = Buffer.from(cropContent, "base64"); + expect(cropBytes.subarray(0, 8)).toEqual(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])); + expect(cropBytes.length).toBeLessThanOrEqual(1024 * 1024); + await writeFile(path.join(proofDir, `${target}-${fixture.format}-request-crop.png`), cropBytes); + // Decode the actual outbound image in Chromium: metadata or a canned blank PNG + // cannot prove that the selected production canvas supplied the attachment. + const crop = await page.evaluate(async (base64) => { + const bytes = Uint8Array.from(atob(base64), (character) => character.charCodeAt(0)); + const image = await createImageBitmap(new Blob([bytes], { type: "image/png" })); + try { + const canvas = document.createElement("canvas"); + canvas.width = image.width; + canvas.height = image.height; + const context = canvas.getContext("2d"); + if (!context) { + throw new Error("crop image has no pixel context"); + } + context.drawImage(image, 0, 0); + let teal = false; + for (let y = 1; y < 12; y++) { + for (let x = 1; x < 12; x++) { + const [red = 255, green = 0, blue = 0, alpha = 0] = context.getImageData( + Math.floor((canvas.width * x) / 12), + Math.floor((canvas.height * y) / 12), + 1, + 1, + ).data; + teal ||= alpha > 240 && red < 70 && green > 90 && blue > 80 && green - red > 70; + } + } + return { width: image.width, height: image.height, teal }; + } finally { + image.close(); + } + }, cropContent); + expect(crop.teal).toBe(true); + expect(crop.width).toBeGreaterThan(0); + expect(crop.height).toBeGreaterThan(0); + expect(crop.width).toBeLessThan(canvasSize.width); + expect(crop.height).toBeLessThan(canvasSize.height); + expect(crop.width / crop.height).toBeCloseTo( + (canvasSize.width * 0.5) / (canvasSize.height * 0.4), + 1, + ); + await viewer.getByText("Request sent", { exact: true }).waitFor(); + expect(await viewer.locator(".viewer").getAttribute("data-request-mode")).toBe("off"); + expect(await viewer.locator(".request-popover").count()).toBe(0); + expect(await viewer.locator(".selection-region").count()).toBe(0); + await expectInline(); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-request-sent.png`), + fullPage: true, + }); + expect(await gateway.getRequests("chat.send")).toHaveLength(1); + console.log( + "document-change-request-proof", + JSON.stringify({ + target, + format: fixture.format, + pageUnit, + pageNumber: 1, + ownedSession: true, + sourceReplyBound: true, + explicitOptIn: true, + pointClickText: true, + inlineRequest: true, + retiredDraftsSent: false, + sends: 1, + cropBytes: cropBytes.length, + cropWidth: crop.width, + cropHeight: crop.height, + }), + ); + await page.setViewportSize(originalViewport); +} diff --git a/ui/src/e2e/platformclaw-document-preview-browser.test-support.ts b/ui/src/e2e/platformclaw-document-preview-browser.test-support.ts new file mode 100644 index 0000000000..135298cfc0 --- /dev/null +++ b/ui/src/e2e/platformclaw-document-preview-browser.test-support.ts @@ -0,0 +1,481 @@ +import path from "node:path"; +import type { Locator, Page } from "playwright"; +import { expect } from "vitest"; +import type { DocumentPreviewFixture } from "./platformclaw-document-preview.test-fixtures.ts"; + +const FAILURE_SCREENSHOT_TIMEOUT_MS = 1_000; + +export async function expectRenderedPage( + viewer: Locator, + current: number, + total: number, + expectAnnotation: boolean, + pageUnit: "page" | "slide" = "page", +): Promise { + await expect + .poll(() => + viewer.evaluate((element, annotationRequired) => { + const canvas = element.shadowRoot?.querySelector("canvas.current-page"); + if (!(canvas instanceof HTMLCanvasElement) || !canvas.width || !canvas.height) { + return null; + } + const context = canvas.getContext("2d"); + if (!context) { + throw new Error("document viewer has no rendered canvas context"); + } + // Both independent fixtures contain teal: a PDF rectangle or a slide + // background. An empty canvas and correct metadata cannot satisfy this. + let teal = false; + let magenta = false; + for (let y = 1; y < 12; y++) { + for (let x = 1; x < 12; x++) { + const pixel = context.getImageData( + Math.floor((canvas.width * x) / 12), + Math.floor((canvas.height * y) / 12), + 1, + 1, + ).data; + const [red = 255, green = 0, blue = 0, alpha = 0] = pixel; + if (alpha > 240 && red < 70 && green > 90 && blue > 80 && green - red > 70) { + teal = true; + } + if (alpha > 240 && red > 230 && green < 30 && blue > 230) { + magenta = true; + } + } + } + return { + state: element.getAttribute("data-render-state"), + current: Number(canvas.getAttribute("data-rendered-page")), + total: Number(canvas.getAttribute("data-page-count")), + teal, + annotation: !annotationRequired || magenta, + }; + }, expectAnnotation), + ) + .toEqual({ state: "ready", current, total, teal: true, annotation: true }); + const position = viewer.locator(".position"); + expect(await position.textContent()).toBe(`${current} / ${total}`); + expect(await position.getAttribute("aria-label")).toBe( + `${pageUnit === "slide" ? "Slide" : "Page"} ${current} of ${total}`, + ); + expect(await viewer.locator(".pagination").textContent()).toContain( + pageUnit === "slide" ? "Slide" : "Page", + ); +} + +async function canvasWidth(viewer: Locator): Promise { + return viewer + .locator("canvas.current-page") + .evaluate((canvas) => canvas.getBoundingClientRect().width); +} + +export async function expectFittedPage(viewer: Locator, fit: "width" | "page"): Promise { + await expect + .poll(() => + viewer.evaluate((element, requestedFit) => { + const stage = element.shadowRoot?.querySelector(".stage"); + const canvas = element.shadowRoot?.querySelector("canvas.current-page"); + if (!(stage instanceof HTMLElement) || !(canvas instanceof HTMLCanvasElement)) { + return false; + } + const style = getComputedStyle(stage); + const availableWidth = + stage.clientWidth - + Number.parseFloat(style.paddingLeft) - + Number.parseFloat(style.paddingRight); + const availableHeight = + stage.clientHeight - + Number.parseFloat(style.paddingTop) - + Number.parseFloat(style.paddingBottom); + const bounds = canvas.getBoundingClientRect(); + const fillsWidth = Math.abs(bounds.width - availableWidth) <= 2; + const fillsHeight = Math.abs(bounds.height - availableHeight) <= 2; + return ( + element.getAttribute("data-render-state") === "ready" && + (requestedFit === "width" + ? fillsWidth + : bounds.width <= availableWidth + 2 && + bounds.height <= availableHeight + 2 && + (fillsWidth || fillsHeight)) + ); + }, fit), + ) + .toBe(true); +} + +export async function clickDocumentPreviewOption(viewer: Locator, name: string): Promise { + const summary = viewer.locator('summary[aria-label="Preview options"]'); + const isOpen = () => + summary.evaluate( + (element) => + element.parentElement instanceof HTMLDetailsElement && element.parentElement.open, + ); + if (!(await isOpen())) { + await summary.click(); + } + await viewer.getByRole("button", { name, exact: true }).click(); + // Expand/collapse can replace the menu along with its canvas. Read the new + // menu instead of keeping hidden advanced controls open in the next proof. + if (await isOpen()) { + await summary.click(); + } +} + +async function expectPageText( + viewer: Locator, + text: string, + pageUnit: "page" | "slide", +): Promise { + const options = viewer.locator('summary[aria-label="Preview options"]'); + await options.click(); + const disclosure = viewer.getByText(pageUnit === "slide" ? "Slide text" : "Page text", { + exact: true, + }); + await disclosure.click(); + const pageText = viewer.locator(".page-text"); + await pageText.waitFor({ state: "visible" }); + expect(await pageText.textContent()).toContain(text); + await disclosure.click(); + await options.click(); +} + +async function exerciseDocumentContents(params: { + page: Page; + viewer: Locator; + target: string; + fixture: DocumentPreviewFixture; + total: number; + proofDir: string; + pageUnit: "page" | "slide"; +}): Promise { + const { page, viewer, target, fixture, total, proofDir, pageUnit } = params; + const contentsButton = viewer.getByRole("button", { name: "Contents", exact: true }); + expect(await contentsButton.getAttribute("aria-expanded")).toBe("false"); + expect(await viewer.locator("openclaw-document-contents").count()).toBe(0); + await contentsButton.click(); + const contents = viewer.locator("openclaw-document-contents"); + await contents.getByRole("navigation", { name: "Contents", exact: true }).waitFor(); + const firstLabel = `${pageUnit === "slide" ? "Slide" : "Page"} 1`; + expect( + await contents + .getByRole("button", { name: firstLabel, exact: true }) + .getAttribute("aria-current"), + ).toBe("page"); + await expect + .poll(() => + contents.locator("canvas").evaluateAll( + (elements) => + elements.length > 0 && + elements.every((element) => { + if ( + !(element instanceof HTMLCanvasElement) || + element.dataset.thumbnailState !== "ready" || + !element.width || + !element.height + ) { + return false; + } + const context = element.getContext("2d"); + if (!context) { + return false; + } + for (let y = 1; y < 12; y++) { + for (let x = 1; x < 12; x++) { + const [red = 255, green = 0, blue = 0, alpha = 0] = context.getImageData( + Math.floor((element.width * x) / 12), + Math.floor((element.height * y) / 12), + 1, + 1, + ).data; + if (alpha > 240 && red < 70 && green > 90 && blue > 80 && green - red > 70) { + return true; + } + } + } + return false; + }), + ), + ) + .toBe(true); + expect(await contents.locator("canvas").count()).toBe(total); + expect(await viewer.getByRole("spinbutton").count()).toBe(0); + if (total > 1) { + const last = contents.getByRole("button", { + name: `${pageUnit === "slide" ? "Slide" : "Page"} ${total}`, + exact: true, + }); + await last.click(); + await expectRenderedPage(viewer, total, total, fixture.format === "pdf", pageUnit); + expect(await last.getAttribute("aria-current")).toBe("page"); + } + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-contents.png`), + fullPage: true, + }); + if (total > 1) { + await contents.getByRole("button", { name: firstLabel, exact: true }).click(); + await expectRenderedPage(viewer, 1, total, fixture.format === "pdf", pageUnit); + await viewer.locator(".stage").press("PageDown"); + await expectRenderedPage(viewer, 2, total, fixture.format === "pdf", pageUnit); + await viewer.locator(".stage").press("PageUp"); + await expectRenderedPage(viewer, 1, total, fixture.format === "pdf", pageUnit); + } + await contentsButton.click(); + await contents.waitFor({ state: "detached" }); +} + +export type ObservedMediaResponse = { + url: string; + status: number; + headers: Record; + bytes: Buffer; +}; + +export async function exerciseDocumentPreview(params: { + page: Page; + fixture: DocumentPreviewFixture; + target: string; + phase: "initial" | "reloaded" | "recovered"; + proofDir: string; + mediaResponses: readonly ObservedMediaResponse[]; + cspViolations: readonly string[]; +}): Promise { + const { page, fixture, target, phase, proofDir, mediaResponses } = params; + if (fixture.format === "image") { + const image = page.locator(".chat-message-image"); + await expect + .poll(() => + image.evaluate((element) => + element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0, + ), + ) + .toBe(320); + await image.click(); + const lightbox = page.locator("openclaw-image-lightbox"); + await lightbox.locator(".image").waitFor(); + await expect + .poll(() => + lightbox + .locator(".image") + .evaluate((element) => + element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0, + ), + ) + .toBe(320); + await page.screenshot({ + path: path.join(proofDir, `${target}-image-${phase}.png`), + fullPage: true, + }); + await lightbox.getByRole("button", { name: "Close image preview" }).click(); + return; + } + const card = page + .locator("openclaw-chat-document-attachment") + .filter({ hasText: fixture.filename }); + const href = await card + .getByRole("link", { name: "Download file", exact: true }) + .getAttribute("href"); + if (!href) { + throw new Error("owned document has no original download URL"); + } + const expectedUrl = new URL(href, page.url()); + if (fixture.format === "powerpoint") { + expectedUrl.searchParams.set("preview", "1"); + } + await card.scrollIntoViewIfNeeded(); + if (phase === "recovered") { + await card.getByRole("button", { name: "Preview", exact: true }).click(); + } + // Initial and reload proof must exercise visibility-triggered opening. The + // response may precede this helper, so use the already-recorded real BFF bytes. + await card.getByRole("button", { name: "Close preview", exact: true }).waitFor(); + await expect + .poll(() => mediaResponses.findLast((candidate) => candidate.url === expectedUrl.href)?.status) + .toBe(200); + const observed = mediaResponses.findLast((candidate) => candidate.url === expectedUrl.href); + if (!observed) { + throw new Error("preview did not traverse the real owned BFF relay"); + } + expect(observed.status).toBe(200); + expect(observed.headers["content-type"]).toMatch(/^application\/pdf(?:;|$)/u); + expect(new URL(observed.url).searchParams.get("preview")).toBe( + fixture.format === "powerpoint" ? "1" : null, + ); + const pdfBytes = observed.bytes; + expect(pdfBytes.subarray(0, 5).toString()).toBe("%PDF-"); + if (fixture.format === "pdf") { + expect(pdfBytes).toEqual(fixture.bytes); + } else { + expect(pdfBytes).not.toEqual(fixture.bytes); + } + const viewer = card.locator("openclaw-document-viewer"); + const total = fixture.format === "powerpoint" ? 2 : 1; + const expectAnnotation = fixture.format === "pdf"; + const pageUnit = fixture.format === "powerpoint" ? "slide" : "page"; + try { + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + expect(await viewer.locator("canvas.current-page").getAttribute("aria-label")).toContain( + fixture.filename, + ); + await expectFittedPage(viewer, "width"); + expect(await viewer.getByRole("button", { name: "Zoom in", exact: true }).isVisible()).toBe( + false, + ); + expect(await viewer.getByRole("button", { name: "Fit page", exact: true }).isVisible()).toBe( + false, + ); + const footer = await viewer.locator(".toolbar").evaluate((element) => { + const layout = element.previousElementSibling; + if (!(layout instanceof HTMLElement) || !layout.classList.contains("page-layout")) { + throw new Error("document controls are not below the preview"); + } + return { + top: element.getBoundingClientRect().top, + pageBottom: layout.getBoundingClientRect().bottom, + }; + }); + expect(footer.top).toBeGreaterThanOrEqual(footer.pageBottom - 1); + await expectPageText( + viewer, + fixture.format === "powerpoint" ? "Owned presentation slide 1" : "Owned document preview", + pageUnit, + ); + expect( + await viewer.getByRole("button", { name: `Previous ${pageUnit}`, exact: true }).isDisabled(), + ).toBe(true); + const next = viewer.getByRole("button", { name: `Next ${pageUnit}`, exact: true }); + expect(await next.isDisabled()).toBe(total === 1); + if (total > 1) { + await next.click(); + await expectRenderedPage(viewer, 2, total, expectAnnotation, pageUnit); + await expectPageText(viewer, "Owned presentation slide 2", pageUnit); + expect(await next.isDisabled()).toBe(true); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-${phase}-page-2.png`), + fullPage: true, + }); + await viewer.getByRole("button", { name: `Previous ${pageUnit}`, exact: true }).click(); + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + } + if (phase === "initial") { + await exerciseDocumentContents({ page, viewer, target, fixture, total, proofDir, pageUnit }); + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + } + expect(params.cspViolations).toEqual([]); + } catch (error) { + try { + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-${phase}-viewer-unavailable.png`), + fullPage: true, + timeout: FAILURE_SCREENSHOT_TIMEOUT_MS, + }); + } catch { + console.log( + "document-preview-failure-screenshot", + JSON.stringify({ target, format: fixture.format, phase, outcome: "unavailable" }), + ); + } + throw error; + } + await card.scrollIntoViewIfNeeded(); + const layout = await viewer.evaluate((element) => { + const column = element.closest(".chat-group-messages"); + const attachmentCard = element.closest(".chat-assistant-attachment-card"); + if (!(column instanceof HTMLElement) || !(attachmentCard instanceof HTMLElement)) { + throw new Error("document preview lost its message column"); + } + const style = getComputedStyle(attachmentCard); + const inset = [ + "padding-left", + "padding-right", + "border-left-width", + "border-right-width", + ].reduce((sum, property) => sum + Number.parseFloat(style.getPropertyValue(property)), 0); + return { width: element.getBoundingClientRect().width, available: column.clientWidth - inset }; + }); + expect(Math.abs(layout.width - layout.available)).toBeLessThanOrEqual(1); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-${phase}.png`), + fullPage: true, + }); + if (phase === "initial") { + const desktopViewport = page.viewportSize(); + if (!desktopViewport) { + throw new Error("document browser proof requires a fixed viewport"); + } + const fittedWidth = await canvasWidth(viewer); + await clickDocumentPreviewOption(viewer, "Zoom in"); + await expect.poll(() => canvasWidth(viewer)).toBeGreaterThan(fittedWidth); + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + const zoomedWidth = await canvasWidth(viewer); + await clickDocumentPreviewOption(viewer, "Zoom out"); + await expect.poll(() => canvasWidth(viewer)).toBeLessThan(zoomedWidth); + await clickDocumentPreviewOption(viewer, "Fit width"); + await expectFittedPage(viewer, "width"); + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + const widthFit = await canvasWidth(viewer); + await clickDocumentPreviewOption(viewer, "Fit page"); + await expectFittedPage(viewer, "page"); + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + expect(await canvasWidth(viewer)).toBeLessThanOrEqual(widthFit + 1); + + await clickDocumentPreviewOption(viewer, "Expand document preview"); + const modal = viewer.locator("openclaw-modal-dialog"); + await modal.waitFor({ state: "attached" }); + await viewer.getByRole("dialog").waitFor({ state: "visible" }); + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + const expanded = await viewer + .locator(".stage") + .evaluate((stage) => stage.getBoundingClientRect().width); + expect(expanded).toBeGreaterThan(layout.width); + expect(expanded).toBeGreaterThan(desktopViewport.width * 0.8); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-expanded.png`), + fullPage: true, + }); + await page.keyboard.press("Escape"); + await modal.waitFor({ state: "detached" }); + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + await expect + .poll(() => + viewer.locator('summary[aria-label="Preview options"]').evaluate((button) => { + const root = button.getRootNode(); + return root instanceof ShadowRoot && root.activeElement === button; + }), + ) + .toBe(true); + + await page.setViewportSize({ width: 390, height: 844 }); + await card.scrollIntoViewIfNeeded(); + await clickDocumentPreviewOption(viewer, "Fit width"); + await expectFittedPage(viewer, "width"); + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + const bounds = await viewer.boundingBox(); + if (!bounds) { + throw new Error("document preview disappeared at mobile width"); + } + expect(bounds.x).toBeGreaterThanOrEqual(0); + expect(bounds.x + bounds.width).toBeLessThanOrEqual(390); + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual( + 391, + ); + await page.screenshot({ + path: path.join(proofDir, `${target}-${fixture.format}-mobile.png`), + fullPage: true, + }); + await page.setViewportSize(desktopViewport); + } + await card.getByRole("button", { name: "Close preview", exact: true }).click(); + // Click completion does not wait for the component's closed-state render. + await card.getByRole("button", { name: "Preview", exact: true }).waitFor(); + expect(await card.locator("openclaw-document-viewer").count()).toBe(0); + if (phase === "initial") { + await card.getByRole("button", { name: "Preview", exact: true }).click(); + await expectRenderedPage(viewer, 1, total, expectAnnotation, pageUnit); + await card.getByRole("button", { name: "Close preview", exact: true }).click(); + await card.getByRole("button", { name: "Preview", exact: true }).waitFor(); + expect(await card.locator("openclaw-document-viewer").count()).toBe(0); + } + expect(params.cspViolations).toEqual([]); +} diff --git a/ui/src/e2e/platformclaw-document-preview.test-fixtures.ts b/ui/src/e2e/platformclaw-document-preview.test-fixtures.ts index 5cf7bd8a51..12317ff5f2 100644 --- a/ui/src/e2e/platformclaw-document-preview.test-fixtures.ts +++ b/ui/src/e2e/platformclaw-document-preview.test-fixtures.ts @@ -50,29 +50,70 @@ function createPreviewPng(): Buffer { ]); } +function createPdf(objects: readonly (string | Buffer)[]): Buffer { + const header = Buffer.from("%PDF-1.4\n% PlatformClaw document fixture\n"); + const chunks: Buffer[] = [header]; + let length = header.length; + const offsets: number[] = []; + for (const [index, object] of objects.entries()) { + offsets.push(length); + const objectChunks = [ + Buffer.from(`${index + 1} 0 obj\n`), + typeof object === "string" ? Buffer.from(object) : object, + Buffer.from("\nendobj\n"), + ]; + chunks.push(...objectChunks); + length += objectChunks.reduce((sum, chunk) => sum + chunk.length, 0); + } + let xref = `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`; + for (const offset of offsets) { + xref += `${String(offset).padStart(10, "0")} 00000 n \n`; + } + xref += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${length}\n%%EOF\n`; + return Buffer.concat([...chunks, Buffer.from(xref)]); +} + +function pdfStream(dictionary: string, content: string | Buffer): Buffer { + const bytes = typeof content === "string" ? Buffer.from(content) : content; + return Buffer.concat([ + Buffer.from(`<< ${dictionary} /Length ${bytes.length} >>\nstream\n`), + bytes, + Buffer.from("\nendstream"), + ]); +} + function createPreviewPdf(): Buffer { const content = "0.06 0.65 0.61 rg 50 560 500 100 re f\nBT /F1 24 Tf 50 700 Td (Owned document preview) Tj 0 -40 Td (PDF snapshot from the selected runtime) Tj ET\n"; - const objects = [ + return createPdf([ "<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R /Annots [6 0 R] >>", "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", - `<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}endstream`, - ]; - let pdf = "%PDF-1.4\n% PlatformClaw owned document fixture\n"; - const offsets: number[] = []; - for (const [index, object] of objects.entries()) { - offsets.push(Buffer.byteLength(pdf)); - pdf += `${index + 1} 0 obj\n${object}\nendobj\n`; - } - const xrefOffset = Buffer.byteLength(pdf); - pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`; - for (const offset of offsets) { - pdf += `${String(offset).padStart(10, "0")} 00000 n \n`; - } - pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xrefOffset}\n%%EOF\n`; - return Buffer.from(pdf, "ascii"); + pdfStream("", content), + // Magenta exists only in this static appearance, so disabling annotation + // rendering cannot pass the real browser's independent pixel assertion. + "<< /Type /Annot /Subtype /Square /Rect [80 360 300 440] /F 4 /AP << /N 7 0 R >> >>", + pdfStream( + "/Type /XObject /Subtype /Form /BBox [0 0 220 80] /Resources << >>", + "1 0 1 rg 0 0 220 80 re f", + ), + ]); +} + +/** Uniform grayscale compresses well while preserving the requested decoded-image dimensions. */ +export function createScanPreviewPdf(width: number, height: number): Buffer { + const compressed = deflateSync(Buffer.alloc(width * height, 128)); + return createPdf([ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 700] /Resources << /XObject << /Scan 5 0 R >> >> /Contents 4 0 R >>", + pdfStream("", "q 500 0 0 700 0 0 cm /Scan Do Q"), + pdfStream( + `/Type /XObject /Subtype /Image /Width ${width} /Height ${height} /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode`, + compressed, + ), + ]); } async function createPreviewPresentation(): Promise { diff --git a/ui/src/e2e/platformclaw-document-preview.test-support.ts b/ui/src/e2e/platformclaw-document-preview.test-support.ts index b827e04fcc..e259528b98 100644 --- a/ui/src/e2e/platformclaw-document-preview.test-support.ts +++ b/ui/src/e2e/platformclaw-document-preview.test-support.ts @@ -2,7 +2,7 @@ import { mkdir, readFile, unlink, writeFile } from "node:fs/promises"; import { createServer, type Server } from "node:http"; import path from "node:path"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; -import { chromium, type Browser, type Page } from "playwright"; +import { chromium, type Browser } from "playwright"; import { afterAll, beforeAll, describe, expect, it, onTestFinished } from "vitest"; import { PlatformClawBrowserMediaRelay } from "../../../packages/platformclaw-control-plane/src/browser-media-http.js"; import { getReplyPayloadMetadata } from "../../../src/auto-reply/reply-payload.js"; @@ -10,10 +10,6 @@ import { buildReplyPayloads } from "../../../src/auto-reply/reply/agent-runner-p import { createReplyMediaPathNormalizer } from "../../../src/auto-reply/reply/reply-media-paths.js"; import { replaceSessionEntry } from "../../../src/config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../../../src/config/types.openclaw.js"; -import { - buildControlUiCspHeader, - computeInlineScriptHashes, -} from "../../../src/gateway/control-ui-csp.js"; import { attachManagedOutgoingMediaToMessage, resolveManagedOutgoingMediaArtifactDownload, @@ -30,12 +26,20 @@ import { withEnvAsync } from "../../../src/test-utils/env.js"; import { createOpenClawTestState } from "../../../src/test-utils/openclaw-test-state.js"; import type { DocumentPreviewExecutionRuntime } from "../../../test/e2e/platformclaw/document-preview-runtime.ts"; import { - installMockGateway, resolvePlaywrightChromiumExecutablePath, - startControlUiE2eServer, type ControlUiE2eServer, } from "../test-helpers/control-ui-e2e.ts"; +import { + installPlatformClawMockGateway, + observePlatformClawPreviewCsp, + startPlatformClawControlUiE2eServer, +} from "../test-helpers/platformclaw-control-ui-e2e.ts"; import { createCanvasExecutionBridge } from "./platformclaw-canvas.test-support.ts"; +import { exerciseDocumentChangeRequest } from "./platformclaw-document-change-browser.test-support.ts"; +import { + exerciseDocumentPreview, + type ObservedMediaResponse, +} from "./platformclaw-document-preview-browser.test-support.ts"; import { createDocumentPreviewFixtures, type DocumentPreviewFixture, @@ -48,13 +52,6 @@ const gatewayToken = "document-preview-gateway-token"; const browserToken = "document-preview-browser-token"; const proofRoot = path.resolve(".artifacts/control-ui-e2e/platformclaw-canvas/document-previews"); -type ObservedMediaResponse = { - url: string; - status: number; - headers: Record; - bytes: Buffer; -}; - function listen(server: Server): Promise { return new Promise((resolve, reject) => { server.once("error", reject); @@ -75,154 +72,6 @@ function close(server: Server): Promise { }); } -async function exerciseDocumentPreview(params: { - page: Page; - fixture: DocumentPreviewFixture; - target: string; - phase: "initial" | "reloaded" | "recovered"; - proofDir: string; - mediaResponses: readonly ObservedMediaResponse[]; -}): Promise { - const { page, fixture, target, phase, proofDir, mediaResponses } = params; - if (fixture.format === "image") { - const image = page.locator(".chat-message-image"); - await expect - .poll(() => - image.evaluate((element) => - element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0, - ), - ) - .toBe(320); - await image.click(); - const lightbox = page.locator("openclaw-image-lightbox"); - await lightbox.locator(".image").waitFor(); - await expect - .poll(() => - lightbox - .locator(".image") - .evaluate((element) => - element instanceof HTMLImageElement && element.complete ? element.naturalWidth : 0, - ), - ) - .toBe(320); - await page.screenshot({ - path: path.join(proofDir, `${target}-image-${phase}.png`), - fullPage: true, - }); - await lightbox.getByRole("button", { name: "Close image preview" }).click(); - return; - } - const card = page - .locator("openclaw-chat-document-attachment") - .filter({ hasText: fixture.filename }); - const href = await card - .getByRole("link", { name: "Download file", exact: true }) - .getAttribute("href"); - if (!href) { - throw new Error("owned document has no original download URL"); - } - const expectedUrl = new URL(href, page.url()); - if (fixture.format === "powerpoint") { - expectedUrl.searchParams.set("preview", "1"); - } - const [previewResponse] = await Promise.all([ - page.waitForResponse((candidate) => candidate.url() === expectedUrl.href), - card.getByRole("button", { name: "Preview", exact: true }).click(), - ]); - expect(previewResponse.status()).toBe(200); - const observed = mediaResponses.findLast((candidate) => candidate.url === expectedUrl.href); - if (!observed) { - throw new Error("preview did not traverse the real owned BFF relay"); - } - expect(observed.status).toBe(200); - expect(observed.headers["content-type"]).toMatch(/^application\/pdf(?:;|$)/u); - expect(new URL(observed.url).searchParams.get("preview")).toBe( - fixture.format === "powerpoint" ? "1" : null, - ); - const pdfBytes = observed.bytes; - expect(pdfBytes.subarray(0, 5).toString()).toBe("%PDF-"); - if (fixture.format === "pdf") { - expect(pdfBytes).toEqual(fixture.bytes); - } else { - expect(pdfBytes).not.toEqual(fixture.bytes); - } - const frame = card.locator("iframe.chat-document-attachment__frame"); - await frame.waitFor(); - const previewBlobUrl = await frame.getAttribute("src"); - expect(previewBlobUrl).toMatch(/^blob:/u); - // Full Chromium's native PDF reader owns these document-load facts. Blob bytes - // and an iframe alone can otherwise pass while the visible page stays blank. - await expect - .poll(async () => { - const viewerFrame = page - .frames() - .find( - (candidate) => - candidate.url().startsWith("chrome-extension://") && - candidate.parentFrame()?.url() === previewBlobUrl, - ); - if (!viewerFrame) { - return null; - } - return viewerFrame.locator("pdf-viewer").evaluate((element) => { - const dimensions: unknown = Reflect.get(element, "documentDimensions"); - const pages = - typeof dimensions === "object" && - dimensions !== null && - "pageDimensions" in dimensions && - Array.isArray(dimensions.pageDimensions) - ? dimensions.pageDimensions.length - : 0; - return { state: Reflect.get(element, "loadState_"), pages }; - }); - }) - .toEqual({ state: "success", pages: fixture.format === "powerpoint" ? 2 : 1 }); - await card.scrollIntoViewIfNeeded(); - const layout = await frame.evaluate((element) => { - const column = element.closest(".chat-group-messages"); - const attachmentCard = element.closest(".chat-assistant-attachment-card"); - if (!(column instanceof HTMLElement) || !(attachmentCard instanceof HTMLElement)) { - throw new Error("document preview lost its message column"); - } - const style = getComputedStyle(attachmentCard); - const inset = [ - "padding-left", - "padding-right", - "border-left-width", - "border-right-width", - ].reduce((total, property) => total + Number.parseFloat(style.getPropertyValue(property)), 0); - return { width: element.getBoundingClientRect().width, available: column.clientWidth - inset }; - }); - expect(Math.abs(layout.width - layout.available)).toBeLessThanOrEqual(1); - await page.screenshot({ - path: path.join(proofDir, `${target}-${fixture.format}-${phase}.png`), - fullPage: true, - }); - if (phase === "initial") { - const desktopViewport = page.viewportSize(); - if (!desktopViewport) { - throw new Error("document browser proof requires a fixed viewport"); - } - await page.setViewportSize({ width: 390, height: 844 }); - await card.scrollIntoViewIfNeeded(); - const bounds = await frame.boundingBox(); - if (!bounds) { - throw new Error("document preview disappeared at mobile width"); - } - expect(bounds.x).toBeGreaterThanOrEqual(0); - expect(bounds.x + bounds.width).toBeLessThanOrEqual(390); - await page.screenshot({ - path: path.join(proofDir, `${target}-${fixture.format}-mobile.png`), - fullPage: true, - }); - await page.setViewportSize(desktopViewport); - } - await card.getByRole("button", { name: "Close preview", exact: true }).click(); - // Click completion does not wait for the component's closed-state render. - await card.getByRole("button", { name: "Preview", exact: true }).waitFor(); - expect(await card.locator("iframe").count()).toBe(0); -} - /** Normal browser proof and the explicit rootless-Docker suite share the same owner pipeline. */ export function defineDocumentPreviewScenarios(params: { formats: readonly DocumentPreviewFormat[]; @@ -240,7 +89,7 @@ export function defineDocumentPreviewScenarios(params: { let controlUi: ControlUiE2eServer; beforeAll(async () => { await mkdir(proofDir, { recursive: true }); - controlUi = await startControlUiE2eServer(); + controlUi = await startPlatformClawControlUiE2eServer("document-preview"); browser = await chromium.launch({ executablePath: resolvePlaywrightChromiumExecutablePath(chromium.executablePath()), }); @@ -307,6 +156,7 @@ export function defineDocumentPreviewScenarios(params: { const downloads: Array<{ fixture: DocumentPreviewFixture; download: ManagedOutgoingMediaArtifactDownload; + messageId: string; }> = []; const publishAuthorities: Array<() => Promise> = []; for (const fixture of fixtures) { @@ -421,7 +271,7 @@ export function defineDocumentPreviewScenarios(params: { artifactId: media.artifactId, }), ).toBeNull(); - downloads.push({ fixture, download }); + downloads.push({ fixture, download, messageId: appended.messageId }); historyMessages.push({ role: "assistant", content, @@ -502,6 +352,7 @@ export function defineDocumentPreviewScenarios(params: { const context = await browser.newContext({ acceptDownloads: true, locale: "en-US", + serviceWorkers: "block", viewport: { width: 1280, height: 900 }, ...(process.env.OPENCLAW_CAPTURE_UI_PROOF === "1" ? { @@ -534,32 +385,16 @@ export function defineDocumentPreviewScenarios(params: { }); await route.fulfill({ response, body: bytes }); }); - await context.route("**/*", async (route) => { - const request = route.request(); - const navigationUrl = new URL(request.url()); - const chatUrl = new URL("chat", controlUi.baseUrl); - if ( - !request.isNavigationRequest() || - navigationUrl.origin !== chatUrl.origin || - navigationUrl.pathname !== chatUrl.pathname || - request.frame().parentFrame() - ) { - await route.fallback(); - return; - } - const response = await route.fetch(); - const body = await response.body(); - const csp = buildControlUiCspHeader({ - inlineScriptHashes: computeInlineScriptHashes(body.toString("utf8")), - }); - await route.fulfill({ - response, - body, - headers: { ...response.headers(), "content-security-policy": csp }, - }); - }); const page = await context.newPage(); - const gatewayMock = await installMockGateway(page, { + const rendererWorkers: string[] = []; + page.on("worker", (worker) => { + const url = new URL(worker.url()); + expect(url.origin).toBe(new URL(controlUi.baseUrl).origin); + expect(rendererWorkers.length).toBeLessThan(32); + rendererWorkers.push(url.pathname); + }); + const cspViolations = await observePlatformClawPreviewCsp(page); + const gatewayMock = await installPlatformClawMockGateway(page, { assistantAgentId: agentId, defaultAgentId: agentId, historyMessages, @@ -583,11 +418,18 @@ export function defineDocumentPreviewScenarios(params: { }, }, }); - await page.goto(`${controlUi.baseUrl}chat`); for (const phase of ["initial", "reloaded"] as const) { - if (phase === "reloaded") { - await page.reload(); - } + const documentResponse = + phase === "initial" + ? await page.goto(`${controlUi.baseUrl}chat`) + : await page.reload(); + expect(documentResponse?.status()).toBe(200); + const csp = documentResponse?.headers()["content-security-policy"] ?? ""; + expect(csp).not.toBe(""); + console.log( + "document-preview-document-csp", + JSON.stringify({ target, phase, csp: csp.slice(0, 4096) }), + ); for (const fixture of fixtures) { await exerciseDocumentPreview({ page, @@ -596,6 +438,7 @@ export function defineDocumentPreviewScenarios(params: { phase, proofDir, mediaResponses, + cspViolations, }); } } @@ -666,7 +509,7 @@ export function defineDocumentPreviewScenarios(params: { expect(denied.status()).toBe(404); await card.getByRole("alert").waitFor(); expect((await card.getByRole("alert").textContent())?.trim()).not.toBe(""); - expect(await card.locator("iframe").count()).toBe(0); + expect(await card.locator("openclaw-document-viewer").count()).toBe(0); await page.screenshot({ path: path.join(proofDir, `${target}-${document.format}-ownership-denied.png`), fullPage: true, @@ -682,6 +525,28 @@ export function defineDocumentPreviewScenarios(params: { phase: "recovered", proofDir, mediaResponses, + cspViolations, + }); + } + expect(cspViolations).toEqual([]); + expect(rendererWorkers.some((pathname) => /pdf\.worker.*\.js$/u.test(pathname))).toBe( + true, + ); + const changeDocument = + downloads.find(({ fixture }) => fixture.format === "powerpoint") ?? + downloads.find(({ fixture }) => fixture.format === "pdf"); + if (changeDocument) { + await exerciseDocumentChangeRequest({ + page, + fixture: changeDocument.fixture, + target, + proofDir, + sessionKey, + gateway: gatewayMock, + source: { + messageId: changeDocument.messageId, + artifactId: changeDocument.download.artifactId, + }, }); } } finally { diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index 50c144f087..9ef7ca9572 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -5866,6 +5866,52 @@ export const en: TranslationMap = { turnCameraOn: "Turn camera on", voiceTranscript: "Voice transcript", }, + documentViewer: { + pageUnit: "Page", + slideUnit: "Slide", + previewOptions: "Preview options", + instructionPlaceholder: "e.g. Change this title", + previousPage: "Previous page", + nextPage: "Next page", + pagePosition: "Page {current} of {total}", + previousSlide: "Previous slide", + nextSlide: "Next slide", + slidePosition: "Slide {current} of {total}", + slideLabel: "{title}, slide {current} of {total}", + slideThumbnail: "Slide {page}", + slideTextAlternative: "Slide text", + contents: "Contents", + pageThumbnail: "Page {page}", + previousContents: "Previous pages", + nextContents: "Next pages", + thumbnailError: "Thumbnail unavailable", + requestChanges: "Request changes", + requestHint: "Click the part you want to change, or drag a region.", + selectedRegion: "Selected region", + selectedText: "Selected text", + instruction: "Describe the change", + submitRequest: "Send request", + cancelRequest: "Cancel request", + requestUnavailable: "The request was not sent. Keep your instructions and try again.", + requestSent: "Request sent", + changeRequestTitle: "Requested changes to {filename} — {page}", + changeRequestRegion: + "Selected region: left {x}%, top {y}%, width {width}%, height {height}%.", + changeRequestCrop: + "The attached image shows the selected region. The original document is referenced in the reply, not reattached.", + requestCropError: + "The selected region could not be captured. Select a smaller region and try again.", + fitPage: "Fit page", + fitWidth: "Fit width", + expand: "Expand document preview", + collapse: "Collapse document preview", + pageLabel: "{title}, page {current} of {total}", + loadError: + "This document could not be displayed. Close and reopen the preview, or download the original file.", + passwordProtected: + "This document requires a password. Download the original file to open it.", + textAlternative: "Page text", + }, attachments: { attachedFile: "Attached file", previewHtml: "Preview HTML", diff --git a/ui/src/pages/chat/chat-pane-lifecycle.ts b/ui/src/pages/chat/chat-pane-lifecycle.ts index 6f625819fe..05ec3019cd 100644 --- a/ui/src/pages/chat/chat-pane-lifecycle.ts +++ b/ui/src/pages/chat/chat-pane-lifecycle.ts @@ -16,12 +16,14 @@ import { } from "../../components/browser/browser-annotation.ts"; import { MODAL_DIALOG_SELECTOR, resolveAsciiShortcutKey } from "../../lib/keyboard-shortcuts.ts"; import { resolveChatPaneObserverRunId } from "../../lib/observer-digest.ts"; +import { readSessionMethodAccess } from "../../lib/session-method-access.ts"; import { sessionPullRequestsForGateway } from "../../lib/session-pull-requests.ts"; import { parseCatalogSessionKey, type CatalogSessionKey } from "../../lib/sessions/catalog-key.ts"; import { resolveSessionKey } from "../../lib/sessions/index.ts"; import { areUiSessionKeysEquivalent } from "../../lib/sessions/session-key.ts"; import { ensureBoardViewElement, ensureWorkboardCardChipElement } from "./board-session-surface.ts"; import { invalidateChatAvatarCache, refreshChatAvatar } from "./chat-avatar.ts"; +import { requiresChatModelSetup } from "./chat-model-setup.ts"; import { ChatPaneSessionCreation } from "./chat-pane-session-create.ts"; import { CHAT_COMPOSER_TEXTAREA_SELECTOR, @@ -36,7 +38,7 @@ import { handlePageGatewayEvent } from "./chat-state-events.ts"; import type { ChatPageHost } from "./chat-state-host.ts"; import { createPageState } from "./chat-state-page.ts"; import { invalidateChatMetadataCache, refreshPageChat } from "./chat-state-refresh.ts"; -import { selectedChatSessionRow } from "./chat-state-route.ts"; +import { resolveChatAgentId, selectedChatSessionRow } from "./chat-state-route.ts"; import { resetChatViewState } from "./chat-view-state.ts"; import { chatAttachmentFromDataUrl } from "./components/chat-attachments.ts"; import { dismissConfirmedActionPopovers } from "./components/chat-message.ts"; @@ -44,6 +46,8 @@ import { toggleSessionWorkspace } from "./components/chat-session-workspace.ts"; import { openChatThreadSearch, closeChatThreadSearch } from "./components/chat-thread.ts"; import { WIDGET_PROMPT_EVENT, type WidgetPromptEventDetail } from "./components/chat-tool-cards.ts"; import { CHAT_COMPOSER_DRAFT_STORAGE_ERROR } from "./composer-persistence.ts"; +import type { ChatDocumentChangeHandler } from "./document-change-contract.ts"; +import { createChatDocumentChangeHandler } from "./document-change-request.ts"; import { exportChatMarkdown } from "./export.ts"; import { admitInitialTurnHandoff, admitInitialUserMessageHandoff } from "./initial-turn-handoff.ts"; import { readChatSessionSnapshot } from "./session-message-cache.ts"; @@ -57,6 +61,15 @@ export abstract class ChatPaneLifecycle extends ChatPaneSessionCreation { // The rendered composer owns access policy; widgets reuse its prepared send // callback instead of bypassing read-only, archived, and setup restrictions. protected widgetPromptSend?: (text: string, sessionKey?: string) => Promise; + protected documentChangeSend?: ChatDocumentChangeHandler; + // Row memoization keeps this identity stable while each request uses the + // latest prepared owner; held callbacks still observe revocation or routing. + protected readonly forwardDocumentChangeRequest: ChatDocumentChangeHandler = ( + source, + request, + signal, + ) => this.documentChangeSend?.(source, request, signal) ?? Promise.resolve({ admitted: false }); + private documentChangeAccess?: () => boolean; protected prepareChatSubmit( state: ChatPageHost, @@ -66,6 +79,49 @@ export abstract class ChatPaneLifecycle extends ChatPaneSessionCreation { ): () => void { const sessionKey = state.sessionKey; const client = state.client; + this.documentChangeAccess = () => { + const agentId = resolveChatAgentId(state); + const agents = this.context.agents.state.agentsList; + const agent = agents?.agents.find((entry) => entry.id === agentId); + const participationBlocked = this.sessionParticipationTracker.resolve({ + catalog: Boolean(catalogKey), + listLoading: state.sessionsLoading, + sessionKey: `${agentId ?? ""}\0${state.sessionKey}`, + session: selectedChatSessionRow(state), + }); + const sendAccess = readSessionMethodAccess(this.context.gateway.snapshot, { + method: "chat.send", + requiredScope: "operator.write", + }); + // Offline intent follows the composer's durable queue; the existing + // outbox and Gateway revalidate authority when delivery resumes. + const mayQueueOffline = + !state.connected && !sendAccess.allowed && sendAccess.cause === "disconnected"; + return ( + canSend && + !catalogKey && + !suggestionViewer && + this.state === state && + state.sessionKey === sessionKey && + state.client === client && + !this.isCurrentSessionArchived(state) && + !participationBlocked && + (sendAccess.allowed || mayQueueOffline) && + !requiresChatModelSetup({ + catalog: false, + connected: state.connected, + agentsLoaded: agents !== null, + selectedAgentFound: agent !== undefined, + agentModel: agent?.model?.primary, + }) + ); + }; + this.documentChangeSend = this.documentChangeAccess() + ? createChatDocumentChangeHandler({ + state, + isCurrent: () => this.state === state && this.documentChangeAccess?.() === true, + }) + : undefined; this.widgetPromptSend = (text, sourceSessionKey) => { if ( !canSend || @@ -568,6 +624,8 @@ export abstract class ChatPaneLifecycle extends ChatPaneSessionCreation { override disconnectedCallback() { this.widgetPromptSend = undefined; + this.documentChangeSend = undefined; + this.documentChangeAccess = undefined; this.clearComposerPrefillAttention(); this.boardProviderLifecycleConnected = false; this.releaseBoardProviderLease(); diff --git a/ui/src/pages/chat/chat-pane-render.ts b/ui/src/pages/chat/chat-pane-render.ts index c73f6bd368..3afb95ab70 100644 --- a/ui/src/pages/chat/chat-pane-render.ts +++ b/ui/src/pages/chat/chat-pane-render.ts @@ -481,6 +481,9 @@ export class ChatPane extends ChatPaneHeader { state.requestUpdate?.(); }, onSend: this.prepareChatSubmit(state, catalogKey, suggestionViewer, canSend), + onDocumentChangeRequest: this.documentChangeSend + ? this.forwardDocumentChangeRequest + : undefined, onCompact: sessionActionCallbacks.onCompact, onOpenSessionCheckpoints: this.embedded ? undefined diff --git a/ui/src/pages/chat/chat-pane.document-changes.test.ts b/ui/src/pages/chat/chat-pane.document-changes.test.ts new file mode 100644 index 0000000000..3e750a208f --- /dev/null +++ b/ui/src/pages/chat/chat-pane.document-changes.test.ts @@ -0,0 +1,171 @@ +/* @vitest-environment jsdom */ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { GatewayBrowserClient } from "../../api/gateway.ts"; +import type { CatalogSessionKey } from "../../lib/sessions/catalog-key.ts"; +import type { SessionCapability } from "../../lib/sessions/index.ts"; +import { createTestChatPane, type TestChatPane } from "./chat-pane.test-support.ts"; +import type { ChatPageHost } from "./chat-state-host.ts"; +import type { ChatDocumentChangeHandler } from "./document-change-contract.ts"; + +type DocumentPane = TestChatPane & { + prepareChatSubmit( + state: ChatPageHost, + catalog: CatalogSessionKey | null, + suggestion: boolean, + canSend: boolean, + ): () => void; + documentChangeSend?: ChatDocumentChangeHandler; + forwardDocumentChangeRequest: ChatDocumentChangeHandler; +}; + +function setup() { + const client = { request: vi.fn() } as unknown as GatewayBrowserClient; + const fixture = createTestChatPane({ + client, + sessions: {} as SessionCapability, + methods: ["chat.send"], + }); + const pane = fixture.pane as DocumentPane; + const { state } = fixture; + const attachment = { + url: "/api/chat/media/outgoing/agent%3Amain%3Acurrent/owned/full", + kind: "document", + label: "report.pdf", + mimeType: "application/pdf", + }; + const message = { + id: "source-message", + role: "assistant", + content: [{ type: "attachment", attachment }], + }; + state.chatMessages = [message]; + state.handleSendChat = vi.fn(async (_text, options) => { + if ( + options && + typeof options === "object" && + "onAdmitted" in options && + typeof options.onAdmitted === "function" + ) { + options.onAdmitted(); + return; + } + throw new Error("Expected outbox admission callback"); + }); + pane.prepareChatSubmit(state, null, false, true); + if (!pane.documentChangeSend) { + throw new Error("Expected prepared document request owner"); + } + const handler = pane.forwardDocumentChangeRequest; + const source: Parameters[0] = { + sessionKey: state.sessionKey, + messageId: message.id, + message, + sourceIdentity: attachment.url, + label: attachment.label, + pageUnit: "page", + }; + const request = { + pageNumber: 1, + selectedText: "Revenue", + instruction: "Clarify this paragraph.", + }; + return { pane, state, source, request, handler, signal: new AbortController().signal }; +} + +afterEach(() => vi.restoreAllMocks()); + +describe("document request pane authority", () => { + it("uses the owning pane rather than the globally focused pane", async () => { + const owner = setup(); + const other = setup(); + owner.pane.active = false; + other.pane.active = true; + expect(await other.handler(owner.source, owner.request, owner.signal)).toMatchObject({ + admitted: false, + }); + expect(await owner.handler(owner.source, owner.request, owner.signal)).toEqual({ + admitted: true, + }); + expect(owner.state.handleSendChat).toHaveBeenCalledOnce(); + expect(other.state.handleSendChat).not.toHaveBeenCalled(); + }); + + it.each(["permission", "session", "scope", "method", "archived", "catalog", "suggestion"])( + "refuses a %s change through a previously prepared callback", + async (change) => { + const test = setup(); + if (change === "permission") { + test.pane.prepareChatSubmit(test.state, null, false, false); + } + if (change === "session") { + test.state.sessionKey = "agent:main:replacement"; + test.pane.prepareChatSubmit(test.state, null, false, true); + } + if (change === "scope" || change === "method") { + const hello = test.pane.context.gateway.snapshot.hello; + if (!hello) { + throw new Error("Expected current Gateway hello"); + } + test.pane.context.gateway.snapshot.hello = { + ...hello, + ...(change === "scope" + ? { auth: { role: "operator", scopes: ["operator.read"] } } + : { features: { methods: [] } }), + }; + } + if (change === "archived") { + test.state.selectedChatSessionArchived = true; + } + if (change === "catalog") { + test.pane.prepareChatSubmit( + test.state, + { catalogId: "external", hostId: "host", threadId: "thread" }, + false, + true, + ); + } + if (change === "suggestion") { + test.pane.prepareChatSubmit(test.state, null, true, true); + } + expect(await test.handler(test.source, test.request, test.signal)).toMatchObject({ + admitted: false, + }); + expect(test.state.handleSendChat).not.toHaveBeenCalled(); + }, + ); + + it("keeps a held row callback usable across ordinary composer rerenders", async () => { + const test = setup(); + test.pane.prepareChatSubmit(test.state, null, false, true); + expect(test.pane.forwardDocumentChangeRequest).toBe(test.handler); + expect(await test.handler(test.source, test.request, test.signal)).toEqual({ admitted: true }); + expect(test.state.handleSendChat).toHaveBeenCalledOnce(); + test.pane.prepareChatSubmit(test.state, null, false, false); + expect(await test.handler(test.source, test.request, test.signal)).toEqual({ admitted: false }); + expect(test.state.handleSendChat).toHaveBeenCalledOnce(); + }); + + it("admits offline intent through the ordinary prepared composer policy", async () => { + const test = setup(); + test.state.connected = false; + test.pane.context.gateway.snapshot.phase = "offline"; + test.pane.context.gateway.snapshot.client = null; + expect(await test.handler(test.source, test.request, test.signal)).toEqual({ admitted: true }); + expect(test.state.handleSendChat).toHaveBeenCalledOnce(); + }); + + it("respects the canonical owned-session proxy authorization capability", async () => { + const test = setup(); + const hello = test.pane.context.gateway.snapshot.hello; + if (!hello) { + throw new Error("Expected current Gateway hello"); + } + test.pane.context.gateway.snapshot.hello = { + ...hello, + auth: { role: "operator", scopes: ["operator.read"] }, + features: { methods: ["chat.send"], capabilities: ["control-ui.server-authorized-methods"] }, + }; + expect(await test.handler(test.source, test.request, test.signal)).toEqual({ admitted: true }); + expect(test.state.handleSendChat).toHaveBeenCalledOnce(); + }); +}); diff --git a/ui/src/pages/chat/chat-send-submit.ts b/ui/src/pages/chat/chat-send-submit.ts index 58b5462423..a1665e7547 100644 --- a/ui/src/pages/chat/chat-send-submit.ts +++ b/ui/src/pages/chat/chat-send-submit.ts @@ -78,6 +78,12 @@ type ChatSendOptions = { runId?: string; actionScope?: ChatSendActionScope; ignoreReplyTarget?: boolean; + /** A verified app action can reply to its own source without changing the composer. */ + replyToId?: string; + /** Explicit app-owned payloads do not consume the human composer's draft. */ + attachments?: readonly ChatAttachment[]; + /** Revalidate the originating UI action after waits and before outbox admission. */ + isSubmissionCurrent?: () => boolean; /** Queue ownership has transferred; delivery failures now belong to outbox Retry. */ onAdmitted?: (submission: { queueId: string; runId: string }) => void; restoreDraft?: boolean; @@ -220,8 +226,9 @@ export async function handleSendChat( let expectedLeafEntryId = actionScope ? actionScope.expectedLeafEntryId : resolveDisplayedLeafEntryId(host as unknown as ChatState); - const attachmentsToSend = - messageOverride == null ? snapshotChatAttachments(host.chatAttachments) : []; + const attachmentsToSend = snapshotChatAttachments( + messageOverride == null ? host.chatAttachments : (opts?.attachments ?? []), + ); const hasAttachments = attachmentsToSend.length > 0; const skillWorkshopRevision = opts?.skillWorkshopRevision; @@ -428,7 +435,7 @@ export async function handleSendChat( const replyTarget = opts?.ignoreReplyTarget ? null : host.chatReplyTarget; // Persisted ids use replyToId; synthetic replies fall back to a quote. - const replyToId = replyTarget?.sourceMessageId?.trim() || undefined; + const replyToId = opts?.replyToId?.trim() || replyTarget?.sourceMessageId?.trim() || undefined; const effectiveMessage = replyTarget && !replyToId ? prependReplyQuote(message, replyTarget) : message; @@ -437,6 +444,9 @@ export async function handleSendChat( opts?.runId ?? chatSubmitKey(host, "message", effectiveMessage, attachmentsToSend, skillWorkshopRevision); await withChatSubmitGuard(host, submitKey, async () => { + if (opts?.isSubmissionCurrent && !opts.isSubmissionCurrent()) { + return; + } if (needsChatHistoryRefresh(host as unknown as ChatState)) { // Terminal delivery can precede its history read, which can also fail. // Reuse or recover that read before fencing the follow-up send. @@ -489,6 +499,9 @@ export async function handleSendChat( return; } } + if (opts?.isSubmissionCurrent && !opts.isSubmissionCurrent()) { + return; + } const cleared = messageOverride == null ? clearSubmittedComposerState(host, previousDraft, attachmentsToSend) diff --git a/ui/src/pages/chat/chat-send.test.ts b/ui/src/pages/chat/chat-send.test.ts index 5d5cf41da4..c38656f49d 100644 --- a/ui/src/pages/chat/chat-send.test.ts +++ b/ui/src/pages/chat/chat-send.test.ts @@ -1263,6 +1263,108 @@ describe("handleSendChat", () => { vi.stubGlobal("sessionStorage", createStorageMock()); }); + it("keeps an app crop in the ordinary failed outbox and Retry without consuming the composer", async () => { + const crop = { + id: "document-crop", + mimeType: "image/png", + fileName: "selection.png", + dataUrl: "data:image/png;base64,c2VsZWN0aW9u", + }; + const draft = { + id: "human-draft", + mimeType: "text/plain", + dataUrl: "data:text/plain;base64,ZHJhZnQ=", + }; + let attempts = 0; + const host = makeHost({ + requestHandlers: { + "chat.history": idleChatHistory(), + "chat.send": (params: unknown) => { + const payload = requireRecord(params, "document correction payload"); + return { runId: payload.idempotencyKey, status: ++attempts === 1 ? "error" : "started" }; + }, + }, + chatMessage: "unfinished human draft", + chatAttachments: [draft], + chatReplyTarget: { messageId: "unrelated", text: "unrelated reply" }, + }); + const onAdmitted = vi.fn(); + await handleSendChat(host, "Correct the selected document region.", { + literalMessage: true, + ignoreReplyTarget: true, + replyToId: "source-message", + attachments: [crop], + onAdmitted, + isSubmissionCurrent: () => true, + }); + const queued = expectDefined(host.chatQueue[0], "failed document correction"); + expect(queued.sendState).toBe("failed"); + expect(queued.attachments).toEqual([crop]); + expect(loadChatComposerSnapshot(host, host.sessionKey)?.queue?.[0]?.attachments).toEqual([ + crop, + ]); + const firstRunId = expectDefined(queued.sendRunId, "initial document correction run"); + expect(firstRunId).toEqual(expect.stringMatching(uuidPattern)); + await retryQueuedChatMessage(host, queued.id); + const sends = host.request.mock.calls.filter(([method]) => method === "chat.send"); + expect(sends).toHaveLength(2); + const firstPayload = requireRecord(sends[0]?.[1], "initial document correction payload"); + const retryPayload = requireRecord(sends[1]?.[1], "retried document correction payload"); + const retryRunId = expectDefined(host.chatRunId, "active retried document correction run"); + expect(firstPayload.idempotencyKey).toBe(firstRunId); + expect(retryPayload.idempotencyKey).toBe(retryRunId); + expect(retryRunId).toEqual(expect.stringMatching(uuidPattern)); + expect(retryRunId).not.toBe(firstRunId); + for (const [, payload] of sends) { + expect(payload).toMatchObject({ + message: "Correct the selected document region.", + replyToId: "source-message", + attachments: [ + { + type: "image", + mimeType: "image/png", + fileName: "selection.png", + content: "c2VsZWN0aW9u", + }, + ], + }); + } + expect(onAdmitted).toHaveBeenCalledOnce(); + expect(host.chatMessage).toBe("unfinished human draft"); + expect(host.chatAttachments).toEqual([draft]); + expect(host.chatReplyTarget?.messageId).toBe("unrelated"); + }); + + it("rechecks app submission authority after the actual history wait before enqueueing", async () => { + const history = createDeferred(); + const host = makeHost({ + requestHandlers: { "chat.history": () => history.promise }, + chatMessage: "keep draft", + }); + const state = asChatPageHost(host); + state.currentSessionId = "source-session"; + const loading = loadChatHistory(state); + let current = true; + const onAdmitted = vi.fn(); + const sending = handleSendChat(host, "Correction request", { + literalMessage: true, + attachments: [{ id: "crop", mimeType: "image/png", dataUrl: "data:image/png;base64,aA==" }], + isSubmissionCurrent: () => current, + onAdmitted, + }); + current = false; + history.resolve({ + messages: [], + sessionInfo: { key: host.sessionKey, sessionId: "source-session", activeLeafEntryId: null }, + }); + await Promise.all([loading, sending]); + expect(host.request).toHaveBeenCalledWith("chat.history", expect.anything()); + expect(host.request).not.toHaveBeenCalledWith("chat.send", expect.anything()); + expect(host.chatQueue).toEqual([]); + expect(onAdmitted).not.toHaveBeenCalled(); + expect(host.chatMessage).toBe("keep draft"); + }); + it("admits one exact contextual turn without consuming the composer or reply target", async () => { const response = createDeferred<{ status: string; runId: string }>(); const host = makeHost({ diff --git a/ui/src/pages/chat/chat-view.ts b/ui/src/pages/chat/chat-view.ts index efc5d5239a..94e90c318f 100644 --- a/ui/src/pages/chat/chat-view.ts +++ b/ui/src/pages/chat/chat-view.ts @@ -66,6 +66,7 @@ import { openChatThreadSearch, closeChatThreadSearch, } from "./components/chat-thread.ts"; +import type { ChatDocumentChangeHandler } from "./document-change-contract.ts"; import type { ChatInputHistoryKeyInput, ChatInputHistoryKeyResult } from "./input-history.ts"; import type { RealtimeTalkConversationEntry } from "./realtime-talk-conversation.ts"; import type { RealtimeTalkCameraDevice } from "./realtime-talk-input.ts"; @@ -209,6 +210,7 @@ export type ChatProps = { localMediaPreviewRoots?: string[]; assistantAttachmentAuthToken?: string | null; resolveArtifactDownload?: ArtifactDownloadResolver; + onDocumentChangeRequest?: ChatDocumentChangeHandler; autoExpandToolCalls?: boolean; attachments?: ChatAttachment[]; getAttachments?: () => ChatAttachment[]; @@ -381,6 +383,7 @@ export function renderChat(props: ChatProps) { localMediaPreviewRoots: props.localMediaPreviewRoots, assistantAttachmentAuthToken: props.assistantAttachmentAuthToken, resolveArtifactDownload: props.resolveArtifactDownload, + onDocumentChangeRequest: props.onDocumentChangeRequest, canvasPluginSurfaceRoute: props.canvasPluginSurfaceRoute, recoverCanvasPluginSurfaceUrl: props.recoverCanvasPluginSurfaceUrl, embedSandboxMode: props.embedSandboxMode, diff --git a/ui/src/pages/chat/components/chat-document-attachment.test.ts b/ui/src/pages/chat/components/chat-document-attachment.test.ts index 28c22959d2..bfb51a05f1 100644 --- a/ui/src/pages/chat/components/chat-document-attachment.test.ts +++ b/ui/src/pages/chat/components/chat-document-attachment.test.ts @@ -6,20 +6,41 @@ import { documentPreviewKind } from "./chat-document-attachment.ts"; const SOURCE = "/api/chat/media/outgoing/agent%3Aemployee%3Amain/owned-file/full"; const PDF = "%PDF-1.7\nowned document"; const fetchMock = vi.fn(); -const revoke = vi.fn(); -const create = vi.fn(() => "blob:http://localhost/owned-pdf"); +const visibilityObservers: DocumentVisibilityObserver[] = []; + +class DocumentVisibilityObserver { + target: Element | undefined; + + constructor( + private readonly callback: (entries: { target: Element; isIntersecting: boolean }[]) => void, + ) { + visibilityObservers.push(this); + } + + observe(target: Element): void { + this.target = target; + } + + disconnect(): void { + this.target = undefined; + } + + setVisible(isIntersecting: boolean): void { + if (this.target) { + this.callback([{ target: this.target, isIntersecting }]); + } + } +} + +// Authorization/fetch ownership is tested here; renderer behavior has its own browser proof. +vi.mock("../../../components/document-viewer.ts", () => ({})); beforeEach(() => { vi.stubGlobal("fetch", fetchMock); - vi.stubGlobal( - "URL", - class extends URL { - static override createObjectURL = create; - static override revokeObjectURL = revoke; - }, - ); - fetchMock.mockResolvedValue( - new Response(PDF, { headers: { "Content-Type": "application/pdf" } }), + vi.stubGlobal("IntersectionObserver", DocumentVisibilityObserver); + visibilityObservers.length = 0; + fetchMock.mockImplementation(() => + Promise.resolve(new Response(PDF, { headers: { "Content-Type": "application/pdf" } })), ); }); @@ -43,7 +64,16 @@ async function mount(mimeType = "application/pdf", label = "report.pdf") { async function open(element: HTMLElement) { element.querySelector("button")?.click(); - await vi.waitFor(() => expect(element.querySelector("iframe")).not.toBeNull()); + await vi.waitFor(() => expect(element.querySelector("openclaw-document-viewer")).not.toBeNull()); +} + +function setVisible(element: HTMLElement, visible: boolean): void { + const card = element.querySelector(".chat-document-attachment"); + const observer = visibilityObservers.find((candidate) => candidate.target === card); + if (!observer) { + throw new Error("document card visibility was not observed"); + } + observer.setVisible(visible); } describe("managed PDF and PowerPoint preview", () => { @@ -63,7 +93,7 @@ describe("managed PDF and PowerPoint preview", () => { expect(documentPreviewKind(mime, name)).toBe(expected); }); - it("reads an authorized PDF only on opening, keeping the original download", async () => { + it("keeps offscreen reads manual and preserves the original download", async () => { const element = await mount(); expect(fetchMock).not.toHaveBeenCalled(); await open(element); @@ -71,15 +101,100 @@ describe("managed PDF and PowerPoint preview", () => { expect(url instanceof URL ? url.href : url).toContain(`${SOURCE}?mediaTicket=owned-ticket`); expect(options).toMatchObject({ credentials: "same-origin", redirect: "error" }); expect(options?.signal).toBeInstanceOf(AbortSignal); - expect(create).toHaveBeenCalledWith(expect.objectContaining({ type: "application/pdf" })); - expect(element.querySelector("iframe")?.src).toBe("blob:http://localhost/owned-pdf"); + const viewer = element.querySelector("openclaw-document-viewer"); + expect(viewer?.bytes).toEqual(new TextEncoder().encode(PDF).buffer); + expect(viewer?.documentTitle).toBe("report.pdf"); expect(element.querySelector("a")?.getAttribute("href")).toBe( `${SOURCE}?mediaTicket=owned-ticket`, ); element.querySelector("button")?.click(); await settleLitElement(element); - expect(element.querySelector("iframe")).toBeNull(); - expect(revoke).toHaveBeenCalledWith("blob:http://localhost/owned-pdf"); + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); + }); + + it.each([ + ["application/pdf", "report.pdf", "page"], + ["application/vnd.ms-powerpoint", "slides.ppt", "slide"], + ])("forwards the owning request callback and %s page unit", async (mime, label, pageUnit) => { + const element = await mount(mime, label); + const handler = vi.fn(async () => ({ admitted: true as const })); + element.requestChanges = handler; + await open(element); + const viewer = element.querySelector("openclaw-document-viewer"); + expect(viewer?.requestChanges).toBe(handler); + expect(viewer?.pageUnit).toBe(pageUnit); + }); + + it.each([ + ["application/pdf", "report.pdf"], + ["application/vnd.ms-powerpoint", "slides.ppt"], + ["application/vnd.openxmlformats-officedocument.presentationml.presentation", "slides.pptx"], + ])("automatically previews visible %s without expanding", async (mime, label) => { + const element = await mount(mime, label); + setVisible(element, false); + await settleLitElement(element); + expect(fetchMock).not.toHaveBeenCalled(); + setVisible(element, true); + await vi.waitFor(() => + expect(element.querySelector("openclaw-document-viewer")).not.toBeNull(), + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(element.querySelector("button")?.getAttribute("aria-expanded")).toBe("true"); + expect(element.querySelector("dialog")).toBeNull(); + setVisible(element, true); + await settleLitElement(element); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("keeps a manually closed source closed through visibility and grant changes", async () => { + const element = await mount(); + setVisible(element, true); + await vi.waitFor(() => + expect(element.querySelector("openclaw-document-viewer")).not.toBeNull(), + ); + element.querySelector("button")?.click(); + await settleLitElement(element); + setVisible(element, false); + element.src = `${SOURCE}?mediaTicket=renewed-ticket`; + element.mimeType = "application/pdf; charset=binary"; + await settleLitElement(element); + setVisible(element, true); + await settleLitElement(element); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); + element.remove(); + document.body.append(element); + await settleLitElement(element); + setVisible(element, true); + await settleLitElement(element); + expect(fetchMock).toHaveBeenCalledTimes(1); + await open(element); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("allows a different visible source to auto-open after manual Close", async () => { + const element = await mount(); + setVisible(element, true); + await vi.waitFor(() => + expect(element.querySelector("openclaw-document-viewer")).not.toBeNull(), + ); + element.querySelector("button")?.click(); + await settleLitElement(element); + const replacement = SOURCE.replace("owned-file", "replacement-file"); + element.sourceIdentity = replacement; + element.src = `${replacement}?mediaTicket=new-owner-ticket`; + await vi.waitFor(() => + expect(element.querySelector("openclaw-document-viewer")).not.toBeNull(), + ); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("retains manual opening when visibility observation is unavailable", async () => { + vi.stubGlobal("IntersectionObserver", undefined); + const element = await mount(); + expect(fetchMock).not.toHaveBeenCalled(); + await open(element); + expect(fetchMock).toHaveBeenCalledTimes(1); }); it("requests PowerPoint conversion on the same signed snapshot route", async () => { @@ -112,8 +227,7 @@ describe("managed PDF and PowerPoint preview", () => { expect(signal?.aborted).toBe(true); finish?.(new Response(PDF, { headers: { "Content-Type": "application/pdf" } })); await settleLitElement(element); - expect(create).not.toHaveBeenCalled(); - expect(element.querySelector("iframe")).toBeNull(); + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); }); it("retires already displayed bytes before accepting a replacement grant", async () => { @@ -122,8 +236,8 @@ describe("managed PDF and PowerPoint preview", () => { fetchMock.mockResolvedValueOnce(new Response("denied", { status: 404 })); element.src = `${SOURCE}?mediaTicket=replaced-ticket`; await vi.waitFor(() => expect(element.querySelector('[role="alert"]')).not.toBeNull()); - expect(revoke).toHaveBeenCalledWith("blob:http://localhost/owned-pdf"); - expect(element.querySelector("iframe")).toBeNull(); + + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); }); it("does not let a cancelled old error hide a newly authorized preview", async () => { @@ -139,11 +253,13 @@ describe("managed PDF and PowerPoint preview", () => { element.querySelector("button")?.click(); await vi.waitFor(() => expect(cancel).toHaveBeenCalled()); element.src = `${SOURCE}?mediaTicket=newly-authorized-ticket`; - await vi.waitFor(() => expect(element.querySelector("iframe")).not.toBeNull()); + await vi.waitFor(() => + expect(element.querySelector("openclaw-document-viewer")).not.toBeNull(), + ); finishCancel?.(); await settleLitElement(element); expect(element.querySelector('[role="alert"]')).toBeNull(); - expect(element.querySelector("iframe")).not.toBeNull(); + expect(element.querySelector("openclaw-document-viewer")).not.toBeNull(); }); it.each([ @@ -157,7 +273,7 @@ describe("managed PDF and PowerPoint preview", () => { element.querySelector("button")?.click(); await vi.waitFor(() => expect(element.querySelector('[role="alert"]')).not.toBeNull()); expect(fetchMock).not.toHaveBeenCalled(); - expect(create).not.toHaveBeenCalled(); + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); }); it.each([ @@ -168,7 +284,7 @@ describe("managed PDF and PowerPoint preview", () => { const element = await mount(); element.querySelector("button")?.click(); await vi.waitFor(() => expect(element.querySelector('[role="alert"]')).not.toBeNull()); - expect(create).not.toHaveBeenCalled(); + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); expect(element.querySelector("a")).not.toBeNull(); }); @@ -187,23 +303,40 @@ describe("managed PDF and PowerPoint preview", () => { element.querySelector("button")?.click(); await vi.waitFor(() => expect(element.textContent).toContain("16 MiB")); expect(cancel).toHaveBeenCalled(); - expect(create).not.toHaveBeenCalled(); + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); }); - it("shows converter enablement guidance and retains the original download", async () => { + it("shows a failed automatic conversion and retains the original download without retries", async () => { fetchMock.mockResolvedValueOnce(new Response("unavailable", { status: 503 })); const element = await mount("application/vnd.ms-powerpoint", "slides.ppt"); - element.querySelector("button")?.click(); + setVisible(element, true); await vi.waitFor(() => expect(element.textContent).toContain("LibreOffice Impress")); - expect(element.querySelector("iframe")).toBeNull(); + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); expect(element.querySelector("a")).not.toBeNull(); + setVisible(element, false); + setVisible(element, true); + await settleLitElement(element); + expect(fetchMock).toHaveBeenCalledTimes(1); }); - it("releases display bytes when its element is removed", async () => { + it("aborts an authorized read when its element is removed", async () => { + let finish: ((response: Response) => void) | undefined; + fetchMock.mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); const element = await mount(); - await open(element); + setVisible(element, true); + await settleLitElement(element); + const signal = fetchMock.mock.calls[0]?.[1]?.signal; element.remove(); - expect(revoke).toHaveBeenCalledWith("blob:http://localhost/owned-pdf"); + expect(signal?.aborted).toBe(true); + expect(visibilityObservers.every((observer) => observer.target === undefined)).toBe(true); + finish?.(new Response(PDF, { headers: { "Content-Type": "application/pdf" } })); + await settleLitElement(element); + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); }); it("aborts a stalled preview and leaves an actionable download outcome", async () => { @@ -224,6 +357,6 @@ describe("managed PDF and PowerPoint preview", () => { expect(element.querySelector('[role="alert"]')?.textContent).toContain("Close and reopen"); expect(element.textContent).not.toContain("private transport detail"); expect(element.querySelector("a")).not.toBeNull(); - expect(create).not.toHaveBeenCalled(); + expect(element.querySelector("openclaw-document-viewer")).toBeNull(); }); }); diff --git a/ui/src/pages/chat/components/chat-document-attachment.ts b/ui/src/pages/chat/components/chat-document-attachment.ts index 0c795bc323..94ad23ce2d 100644 --- a/ui/src/pages/chat/components/chat-document-attachment.ts +++ b/ui/src/pages/chat/components/chat-document-attachment.ts @@ -1,9 +1,11 @@ import { html, nothing, type PropertyValues } from "lit"; import { property, state } from "lit/decorators.js"; +import type { DocumentChangeHandler } from "../../../components/document-viewer-request.ts"; import { icons } from "../../../components/icons.ts"; import { t } from "../../../i18n/index.ts"; import { OpenClawLightDomElement } from "../../../lit/openclaw-element.ts"; import { safeAttachmentHref } from "./chat-attachment-href.ts"; +import "../../../components/document-viewer.ts"; const PREVIEW_MAX_BYTES = 16 * 1024 * 1024; const PREVIEW_TIMEOUT_MS = 30_000; @@ -73,20 +75,35 @@ class ChatDocumentAttachment extends OpenClawLightDomElement { @property() mimeType = ""; @property({ type: Number }) sizeBytes: number | undefined; @property({ attribute: false }) onMediaLoaded: (() => void) | undefined; + @property({ attribute: false }) requestChanges: DocumentChangeHandler | undefined; @state() private opened = false; - @state() private previewUrl = ""; + @state() private previewBytes: ArrayBuffer | undefined; @state() private error = ""; private controller: AbortController | undefined; + private visibilityObserver: IntersectionObserver | undefined; + private visible = false; + private autoPreviewSuppressed = false; + + override connectedCallback(): void { + super.connectedCallback(); + void this.updateComplete.then(() => this.observeVisibility()); + } override disconnectedCallback(): void { + this.visibilityObserver?.disconnect(); + this.visibilityObserver = undefined; + this.visible = false; this.opened = false; this.releasePreview(); super.disconnectedCallback(); } protected override willUpdate(changed: PropertyValues): void { + if (changed.has("sourceIdentity")) { + this.autoPreviewSuppressed = false; + } if (changed.has("sourceIdentity") || changed.has("mimeType")) { this.opened = false; this.releasePreview(); @@ -96,13 +113,58 @@ class ChatDocumentAttachment extends OpenClawLightDomElement { } } + protected override updated(): void { + this.openVisiblePreview(); + } + + private observeVisibility(): void { + if ( + !this.isConnected || + this.visibilityObserver || + typeof IntersectionObserver !== "function" + ) { + return; + } + const card = this.querySelector(".chat-document-attachment"); + if (!card) { + return; + } + const observer = new IntersectionObserver((entries) => { + if (this.visibilityObserver !== observer || !this.isConnected) { + return; + } + for (const entry of entries) { + if (entry.target === card) { + this.visible = entry.isIntersecting; + } + } + this.openVisiblePreview(); + }); + this.visibilityObserver = observer; + observer.observe(card); + } + + private openVisiblePreview(): void { + if ( + !this.isConnected || + !this.visible || + this.opened || + this.autoPreviewSuppressed || + !this.src || + !documentPreviewKind(this.mimeType, this.label) + ) { + return; + } + // Visibility starts the existing owned read; it never opens the expanded viewer. + this.opened = true; + void this.loadPreview(); + this.onMediaLoaded?.(); + } + private releasePreview(): void { this.controller?.abort(); this.controller = undefined; - if (this.previewUrl) { - URL.revokeObjectURL(this.previewUrl); - this.previewUrl = ""; - } + this.previewBytes = undefined; } private async loadPreview(): Promise { @@ -162,8 +224,8 @@ class ChatDocumentAttachment extends OpenClawLightDomElement { return; } controller.signal.throwIfAborted(); - // The browser's PDF reader receives PDF bytes only, never a remote URL or HTML. - this.previewUrl = URL.createObjectURL(new Blob([bytes], { type: "application/pdf" })); + // PDF and converted slides share the same local renderer and authorized snapshot. + this.previewBytes = bytes; } catch (error) { if (this.controller === controller && this.opened && this.isConnected) { this.error = @@ -181,6 +243,8 @@ class ChatDocumentAttachment extends OpenClawLightDomElement { private readonly togglePreview = (): void => { this.opened = !this.opened; + // A renewed grant or another intersection must respect the user's Close action. + this.autoPreviewSuppressed = !this.opened; if (this.opened) { void this.loadPreview(); } else { @@ -216,27 +280,22 @@ class ChatDocumentAttachment extends OpenClawLightDomElement { : nothing} - ${this.opened && documentPreviewKind(this.mimeType, this.label) === "powerpoint" - ? html`
- ${t("chat.attachments.powerPointStaticPreview")} -
` - : nothing} ${this.opened ? this.error ? html`` - : this.previewUrl - ? html`` + : this.previewBytes + ? html`` : html`
${t("common.loading")}
` : nothing} diff --git a/ui/src/pages/chat/components/chat-message-attachment-card.test.ts b/ui/src/pages/chat/components/chat-message-attachment-card.test.ts new file mode 100644 index 0000000000..660f409e81 --- /dev/null +++ b/ui/src/pages/chat/components/chat-message-attachment-card.test.ts @@ -0,0 +1,76 @@ +/* @vitest-environment jsdom */ +import { render } from "lit"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { ChatDocumentChangeHandler } from "../document-change-contract.ts"; +import { renderAssistantDocumentAttachment } from "./chat-message-attachment-card.ts"; + +vi.mock("../../../components/document-viewer.ts", () => ({})); +afterEach(() => document.body.replaceChildren()); + +describe("document attachment request owner", () => { + it("binds the original message and locator rather than the temporary download grant", async () => { + const callback = vi.fn(async () => ({ admitted: true })); + const source = "/api/chat/media/outgoing/agent%3Aemployee%3Amain/artifact/full"; + const message = { id: "owned-message", role: "assistant" }; + const container = document.createElement("div"); + document.body.append(container); + render( + renderAssistantDocumentAttachment({ + url: `${source}?mediaTicket=private-ticket`, + attachment: { + url: source, + kind: "document", + label: "slides.pptx", + artifactId: "artifact", + mimeType: "application/vnd.ms-powerpoint", + }, + options: { + sessionKey: "agent:employee:main", + messageId: message.id, + sourceMessage: message, + onDocumentChangeRequest: callback, + }, + }), + container, + ); + const card = container.querySelector("openclaw-chat-document-attachment"); + if (!card?.requestChanges) { + throw new Error("Missing owned document request callback"); + } + const request = { pageNumber: 2, selectedText: "Revenue", instruction: "Clarify this." }; + const signal = new AbortController().signal; + await card.requestChanges(request, signal); + expect(callback).toHaveBeenCalledExactlyOnceWith( + { + sessionKey: "agent:employee:main", + messageId: "owned-message", + message, + sourceIdentity: source, + artifactId: "artifact", + label: "slides.pptx", + pageUnit: "slide", + }, + request, + signal, + ); + }); + + it("withholds requests when a row cannot identify its owning message", () => { + const container = document.createElement("div"); + render( + renderAssistantDocumentAttachment({ + url: "/api/chat/media/outgoing/agent%3Aemployee%3Amain/artifact/full", + attachment: { + url: "/api/chat/media/outgoing/agent%3Aemployee%3Amain/artifact/full", + kind: "document", + label: "report.pdf", + }, + options: { onDocumentChangeRequest: vi.fn() }, + }), + container, + ); + const card = container.querySelector("openclaw-chat-document-attachment"); + expect(card).not.toBeNull(); + expect(card?.requestChanges).toBeUndefined(); + }); +}); diff --git a/ui/src/pages/chat/components/chat-message-attachment-card.ts b/ui/src/pages/chat/components/chat-message-attachment-card.ts index 3ed88770f6..0cb2a3e810 100644 --- a/ui/src/pages/chat/components/chat-message-attachment-card.ts +++ b/ui/src/pages/chat/components/chat-message-attachment-card.ts @@ -1,4 +1,5 @@ import { html } from "lit"; +import type { DocumentChangeHandler } from "../../../components/document-viewer-request.ts"; import { icons } from "../../../components/icons.ts"; import { safeAttachmentHref } from "./chat-attachment-href.ts"; import { documentPreviewKind } from "./chat-document-attachment.ts"; @@ -24,12 +25,34 @@ export function renderAssistantDocumentAttachment(params: { >`; } if (documentPreviewKind(attachment.mimeType, attachment.label)) { + const pageUnit = + documentPreviewKind(attachment.mimeType, attachment.label) === "powerpoint" + ? "slide" + : "page"; + const requestChanges = options?.onDocumentChangeRequest; + const source = + options?.sessionKey && options.messageId && options.sourceMessage + ? ({ + sessionKey: options.sessionKey, + messageId: options.messageId, + message: options.sourceMessage, + sourceIdentity: attachment.url, + artifactId: attachment.artifactId, + label: attachment.label, + pageUnit, + } as const) + : undefined; + const handler: DocumentChangeHandler | undefined = + requestChanges && source + ? (request, signal) => requestChanges(source, request, signal) + : undefined; return html``; } diff --git a/ui/src/pages/chat/components/chat-message-bubble.ts b/ui/src/pages/chat/components/chat-message-bubble.ts index 0db88033df..35888f455a 100644 --- a/ui/src/pages/chat/components/chat-message-bubble.ts +++ b/ui/src/pages/chat/components/chat-message-bubble.ts @@ -29,6 +29,7 @@ import { } from "../../../lib/chat/tool-cards.ts"; import type { CanvasPluginSurfaceRoute, EmbedSandboxMode } from "../../../lib/chat/tool-display.ts"; import { resolveToolDisplay } from "../../../lib/chat/tool-display.ts"; +import type { ChatDocumentChangeHandler } from "../document-change-contract.ts"; import { visibleWorkspaceConflictPaths, workspaceConflictCount, @@ -185,6 +186,7 @@ export function renderGroupedMessage( localMediaPreviewRoots?: readonly string[]; assistantAttachmentAuthToken?: string | null; resolveArtifactDownload?: ArtifactDownloadResolver; + onDocumentChangeRequest?: ChatDocumentChangeHandler; onAssistantAttachmentLoaded?: () => void; onRequestOpenImage?: () => number; onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void; @@ -224,6 +226,8 @@ export function renderGroupedMessage( onRequestOpenImage: opts.onRequestOpenImage, onOpenImage: opts.onOpenImage, resolveArtifactDownload: opts.resolveArtifactDownload, + onDocumentChangeRequest: opts.onDocumentChangeRequest, + sourceMessage: message, }; schedulePairingQrExpiryRefresh(messageKey, message, opts.onRequestUpdate); const images = resolveRenderableMessageImages(extractImages(message), imageRenderOptions); diff --git a/ui/src/pages/chat/components/chat-message-group.ts b/ui/src/pages/chat/components/chat-message-group.ts index e43be0dd5b..021635552a 100644 --- a/ui/src/pages/chat/components/chat-message-group.ts +++ b/ui/src/pages/chat/components/chat-message-group.ts @@ -18,6 +18,7 @@ import { persistedMessageEntryId, type AssistantMessageExpansionState, } from "../chat-thread.ts"; +import type { ChatDocumentChangeHandler } from "../document-change-contract.ts"; import { workspaceResultConflictFromTranscript } from "../workspace-conflict.ts"; import { renderChatAuthorAvatar } from "./chat-author-avatar.ts"; import { renderGroupedMessage } from "./chat-message-bubble.ts"; @@ -87,6 +88,7 @@ type RenderMessageGroupOptions = { localMediaPreviewRoots?: readonly string[]; assistantAttachmentAuthToken?: string | null; resolveArtifactDownload?: ArtifactDownloadResolver; + onDocumentChangeRequest?: ChatDocumentChangeHandler; canvasPluginSurfaceRoute?: CanvasPluginSurfaceRoute; recoverCanvasPluginSurfaceUrl?: (observedUrl: string) => Promise; embedSandboxMode?: EmbedSandboxMode; @@ -160,6 +162,7 @@ function buildGroupedMessageRenderOptions( localMediaPreviewRoots: opts.localMediaPreviewRoots, assistantAttachmentAuthToken: opts.assistantAttachmentAuthToken, resolveArtifactDownload: opts.resolveArtifactDownload, + onDocumentChangeRequest: opts.onDocumentChangeRequest, embedSandboxMode: opts.embedSandboxMode, allowExternalEmbedUrls: opts.allowExternalEmbedUrls, }; diff --git a/ui/src/pages/chat/components/chat-message-media.ts b/ui/src/pages/chat/components/chat-message-media.ts index 96de299fd0..f6d240cd8a 100644 --- a/ui/src/pages/chat/components/chat-message-media.ts +++ b/ui/src/pages/chat/components/chat-message-media.ts @@ -7,6 +7,7 @@ import { getMediaFileExtension, hasVideoMediaFileExtension, } from "../../../lib/media-file-extension.ts"; +import type { ChatDocumentChangeHandler } from "../document-change-contract.ts"; export type PairingQrExpiryNotice = { title: string; @@ -39,6 +40,8 @@ export type ImageRenderOptions = { onRequestOpenImage?: () => number; onOpenImage?: (item: ImageLightboxItem, requestVersion?: number) => void; resolveArtifactDownload?: ArtifactDownloadResolver; + onDocumentChangeRequest?: ChatDocumentChangeHandler; + sourceMessage?: unknown; }; export type RenderableImageBlock = ImageBlock & { diff --git a/ui/src/pages/chat/components/chat-thread.ts b/ui/src/pages/chat/components/chat-thread.ts index c18128b2b9..719aff703f 100644 --- a/ui/src/pages/chat/components/chat-thread.ts +++ b/ui/src/pages/chat/components/chat-thread.ts @@ -71,6 +71,7 @@ import { syncToolCardExpansionState, } from "../chat-thread.ts"; import { DeletedMessages } from "../deleted-messages.ts"; +import type { ChatDocumentChangeHandler } from "../document-change-contract.ts"; import { PinnedMessages } from "../pinned-messages.ts"; import type { RealtimeTalkConversationEntry } from "../realtime-talk-conversation.ts"; import { @@ -167,6 +168,7 @@ type ChatThreadProps = { localMediaPreviewRoots?: string[]; assistantAttachmentAuthToken?: string | null; resolveArtifactDownload?: ArtifactDownloadResolver; + onDocumentChangeRequest?: ChatDocumentChangeHandler; canvasPluginSurfaceRoute?: CanvasPluginSurfaceRoute; recoverCanvasPluginSurfaceUrl?: (observedUrl: string) => Promise; embedSandboxMode?: EmbedSandboxMode; @@ -1875,6 +1877,7 @@ function renderChatThreadContents( localMediaPreviewRoots: props.localMediaPreviewRoots ?? [], assistantAttachmentAuthToken: props.assistantAttachmentAuthToken ?? null, resolveArtifactDownload: props.resolveArtifactDownload, + onDocumentChangeRequest: props.onDocumentChangeRequest, canvasPluginSurfaceRoute: props.canvasPluginSurfaceRoute, recoverCanvasPluginSurfaceUrl: props.recoverCanvasPluginSurfaceUrl, embedSandboxMode: props.embedSandboxMode ?? "scripts", @@ -2106,6 +2109,7 @@ function renderChatThreadContents( props.allowExternalEmbedUrls ?? false, threadContextWindow, props.onSetReply, + props.onDocumentChangeRequest, turnRecap === null ? "" : `${turnRecap.runtimeMs}:${turnRecap.outputTokens ?? ""}`, ]); const transcriptContents = diff --git a/ui/src/pages/chat/document-change-contract.ts b/ui/src/pages/chat/document-change-contract.ts new file mode 100644 index 0000000000..e8dbc0d577 --- /dev/null +++ b/ui/src/pages/chat/document-change-contract.ts @@ -0,0 +1,20 @@ +import type { + DocumentChangeRequest, + DocumentChangeResult, +} from "../../components/document-viewer-request.ts"; + +export type ChatDocumentSource = { + sessionKey: string; + messageId: string; + message: unknown; + sourceIdentity: string; + artifactId?: string; + label: string; + pageUnit: "page" | "slide"; +}; + +export type ChatDocumentChangeHandler = ( + source: ChatDocumentSource, + request: DocumentChangeRequest, + signal: AbortSignal, +) => Promise; diff --git a/ui/src/pages/chat/document-change-request.test.ts b/ui/src/pages/chat/document-change-request.test.ts new file mode 100644 index 0000000000..f3e18be7d7 --- /dev/null +++ b/ui/src/pages/chat/document-change-request.test.ts @@ -0,0 +1,235 @@ +/* @vitest-environment jsdom */ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DocumentChangeRequest } from "../../components/document-viewer-request.ts"; +import type { ChatAttachment } from "../../lib/chat/chat-types.ts"; +import { + getChatAttachmentDataUrl, + releaseChatAttachmentPayloads, +} from "./attachment-payload-store.ts"; +import { createChatDocumentChangeHandler } from "./document-change-request.ts"; + +const PNG = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jz1kAAAAASUVORK5CYII="; +const payloads: ChatAttachment[] = []; +afterEach(() => releaseChatAttachmentPayloads(payloads.splice(0))); + +function isAttachment(value: unknown): value is ChatAttachment { + return ( + typeof value === "object" && + value !== null && + "id" in value && + typeof value.id === "string" && + "mimeType" in value && + typeof value.mimeType === "string" + ); +} + +function submission(value: unknown) { + if ( + !value || + typeof value !== "object" || + !("onAdmitted" in value) || + !("isSubmissionCurrent" in value) || + !("attachments" in value) || + typeof value.onAdmitted !== "function" || + typeof value.isSubmissionCurrent !== "function" || + !Array.isArray(value.attachments) + ) { + throw new Error("Expected document request submission"); + } + const attachments = value.attachments.filter(isAttachment); + payloads.push(...attachments); + return { onAdmitted: value.onAdmitted, isCurrent: value.isSubmissionCurrent, attachments }; +} + +function fixture( + send: (text: string | undefined, options: unknown) => Promise, + sessionKey = "agent:employee:main", +) { + const attachment = { + url: "/api/chat/media/outgoing/agent%3Aemployee%3Amain/snapshot-id/full", + kind: "document", + label: "slides.pptx", + artifactId: "owned-artifact", + mimeType: "application/vnd.openxmlformats-officedocument.presentationml.presentation", + }; + const message = { + role: "assistant", + id: "source-message", + content: [{ type: "attachment", attachment }], + }; + const state = { + sessionKey, + assistantAgentId: "employee", + currentSessionId: "source-session", + client: null, + connectionEpoch: 1, + chatMessages: [message], + chatError: null, + handleSendChat: vi.fn(send), + }; + const source: Parameters>[0] = { + sessionKey: state.sessionKey, + messageId: message.id, + message, + sourceIdentity: attachment.url, + artifactId: attachment.artifactId, + label: attachment.label, + pageUnit: "slide", + }; + const request: DocumentChangeRequest = { + pageNumber: 1, + region: { x: 0.1, y: 0.2, width: 0.3, height: 0.4 }, + selectedText: "Quarterly revenue", + instruction: "Make this heading clearer.", + cropDataUrl: PNG, + }; + const controller = new AbortController(); + let allowed = true; + const handler = createChatDocumentChangeHandler({ state, isCurrent: () => allowed }); + return { + state, + source, + request, + controller, + handler, + attachment, + deny: () => { + allowed = false; + }, + }; +} + +describe("document change request admission", () => { + it("hands a bounded source reference and crop to the outbox before delivery finishes", async () => { + let finish!: () => void; + const delivery = new Promise((resolve) => { + finish = resolve; + }); + const test = fixture(async (_text, options) => { + const queued = submission(options); + expect(queued.isCurrent()).toBe(true); + queued.onAdmitted(); + await delivery; + }); + Object.assign(test.request.region!, { unrelated: "must not enter the prompt" }); + expect(await test.handler(test.source, test.request, test.controller.signal)).toEqual({ + admitted: true, + }); + const [text, options] = test.state.handleSendChat.mock.calls[0]!; + expect(text).toContain("slides.pptx"); + expect(text).toContain("Slide 1"); + expect(text).toContain("> Quarterly revenue"); + expect(text).toContain("Make this heading clearer."); + expect(text).toContain("original document is referenced in the reply, not reattached"); + expect(text).not.toContain("/api/chat/media/outgoing/"); + expect(text).not.toContain("source-message"); + expect(text).not.toContain("owned-artifact"); + expect(text).not.toContain("Source:"); + expect(text).not.toContain("must not enter the prompt"); + expect(options).toMatchObject({ + literalMessage: true, + ignoreReplyTarget: true, + replyToId: "source-message", + }); + const sent = submission(options); + expect(sent.attachments).toHaveLength(1); + expect(sent.attachments[0]).toMatchObject({ + fileName: "document-selection-slide-1.png", + mimeType: "image/png", + }); + expect(getChatAttachmentDataUrl(sent.attachments[0]!)).toBe(PNG); + finish(); + }); + + it.each([ + "source", + "artifact", + "message", + "session", + "transcript", + "connection", + "permission", + "cancel", + ])("refuses a stale %s without submitting", async (change) => { + const test = fixture(async () => undefined); + if (change === "source") { + test.attachment.url = "/api/chat/media/outgoing/agent%3Aemployee%3Amain/replaced/full"; + } + if (change === "artifact") { + test.source.artifactId = "foreign-artifact"; + } + if (change === "message") { + test.state.chatMessages = []; + } + if (change === "session") { + test.state.sessionKey = "agent:employee:other"; + } + if (change === "transcript") { + test.state.currentSessionId = "replacement-session"; + } + if (change === "connection") { + test.state.connectionEpoch++; + } + if (change === "permission") { + test.deny(); + } + if (change === "cancel") { + test.controller.abort(); + } + expect(await test.handler(test.source, test.request, test.controller.signal)).toMatchObject({ + admitted: false, + }); + expect(test.state.handleSendChat).not.toHaveBeenCalled(); + }); + + it.each([ + "missing coordinate", + "oversized instruction", + "oversized text", + "wrong crop type", + "oversized crop", + ])("rejects %s before allocating an attachment", async (change) => { + const test = fixture(async () => undefined); + if (change === "missing coordinate") { + Reflect.deleteProperty(test.request.region!, "width"); + } + if (change === "oversized instruction") { + test.request.instruction = "x".repeat(4097); + } + if (change === "oversized text") { + test.request.selectedText = "x".repeat(1025); + } + if (change === "wrong crop type") { + test.request.cropDataUrl = "data:image/jpeg;base64,aA=="; + } + if (change === "oversized crop") { + test.request.cropDataUrl = `data:image/png;base64,iVBORw0KGgo${"A".repeat(1_400_000)}`; + } + expect(await test.handler(test.source, test.request, test.controller.signal)).toMatchObject({ + admitted: false, + }); + expect(test.state.handleSendChat).not.toHaveBeenCalled(); + }); + + it("releases an unadmitted crop while retaining the viewer's retryable input", async () => { + const test = fixture(async (_text, options) => { + submission(options); + }); + expect(await test.handler(test.source, test.request, test.controller.signal)).toMatchObject({ + admitted: false, + }); + expect(getChatAttachmentDataUrl(payloads[0]!)).toBeNull(); + expect(test.request.cropDataUrl).toBe(PNG); + expect(test.request.instruction).toBe("Make this heading clearer."); + }); + + it("rejects a global-alias Agent change even when the session spelling stays the same", async () => { + const test = fixture(async () => undefined, "global"); + test.state.assistantAgentId = "other"; + expect(await test.handler(test.source, test.request, test.controller.signal)).toMatchObject({ + admitted: false, + }); + expect(test.state.handleSendChat).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/src/pages/chat/document-change-request.ts b/ui/src/pages/chat/document-change-request.ts new file mode 100644 index 0000000000..10e7407438 --- /dev/null +++ b/ui/src/pages/chat/document-change-request.ts @@ -0,0 +1,208 @@ +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { + DOCUMENT_CHANGE_CROP_MAX_BYTES, + DOCUMENT_CHANGE_INSTRUCTION_MAX, + DOCUMENT_CHANGE_TEXT_MAX, + type DocumentChangeRequest, +} from "../../components/document-viewer-request.ts"; +import { t } from "../../i18n/index.ts"; +import { normalizeMessage } from "../../lib/chat/message-normalizer.ts"; +import { + areUiSessionKeysEquivalent, + resolveUiSelectedSessionAgentId, + type UiSessionDefaultsHost, +} from "../../lib/sessions/session-key.ts"; +import { releaseChatAttachmentPayloads } from "./attachment-payload-store.ts"; +import type { ChatPageHost } from "./chat-state-host.ts"; +import { persistedMessageEntryId } from "./chat-thread-items.ts"; +import { chatAttachmentFromDataUrl } from "./components/chat-attachments.ts"; +import { extractTranscriptAttachments } from "./components/chat-message-media.ts"; +import type { ChatDocumentChangeHandler, ChatDocumentSource } from "./document-change-contract.ts"; + +type DocumentChangeHost = UiSessionDefaultsHost & + Pick< + ChatPageHost, + | "sessionKey" + | "currentSessionId" + | "client" + | "connectionEpoch" + | "chatMessages" + | "handleSendChat" + >; + +function sourceIsCurrent(state: DocumentChangeHost, source: ChatDocumentSource): boolean { + if ( + !source.messageId || + source.messageId.length > 256 || + (source.artifactId?.length ?? 0) > 256 || + !areUiSessionKeysEquivalent(state.sessionKey, source.sessionKey) || + !state.chatMessages.includes(source.message) || + persistedMessageEntryId(source.message) !== source.messageId + ) { + return false; + } + const attachments = [ + ...normalizeMessage(source.message).content.filter((item) => item.type === "attachment"), + ...extractTranscriptAttachments(source.message), + ]; + return attachments.some( + ({ attachment }) => + attachment.url === source.sourceIdentity && + attachment.artifactId === source.artifactId && + attachment.label === source.label, + ); +} + +function validRequest(request: DocumentChangeRequest): boolean { + const { region, cropDataUrl, selectedText, instruction, pageNumber } = request; + if ( + !Number.isSafeInteger(pageNumber) || + pageNumber < 1 || + !instruction.trim() || + instruction.length > DOCUMENT_CHANGE_INSTRUCTION_MAX || + (selectedText?.length ?? 0) > DOCUMENT_CHANGE_TEXT_MAX || + (!region && !selectedText?.trim()) + ) { + return false; + } + if ( + region && + (![region.x, region.y, region.width, region.height].every(Number.isFinite) || + region.x < 0 || + region.y < 0 || + region.width <= 0 || + region.height <= 0 || + region.x + region.width > 1 || + region.y + region.height > 1 || + !cropDataUrl) + ) { + return false; + } + if (cropDataUrl) { + const maxCharacters = Math.ceil(DOCUMENT_CHANGE_CROP_MAX_BYTES / 3) * 4; + const payload = cropDataUrl.slice("data:image/png;base64,".length); + if ( + payload.length > maxCharacters || + !/^data:image\/png;base64,iVBORw0KGgo[A-Za-z0-9+/]*={0,2}$/u.test(cropDataUrl) + ) { + return false; + } + try { + if (atob(payload).length > DOCUMENT_CHANGE_CROP_MAX_BYTES) { + return false; + } + } catch { + return false; + } + } + return true; +} + +/** The existing outbox owns the crop and Retry only after onAdmitted fires. */ +export function createChatDocumentChangeHandler(params: { + state: DocumentChangeHost; + isCurrent: () => boolean; +}): ChatDocumentChangeHandler { + const { state } = params; + const sessionKey = state.sessionKey; + const sessionId = state.currentSessionId; + const client = state.client; + const connectionEpoch = state.connectionEpoch; + const agentId = resolveUiSelectedSessionAgentId(state); + return async (source, request, signal) => { + const isCurrent = () => + !signal.aborted && + params.isCurrent() && + state.sessionKey === sessionKey && + state.currentSessionId === sessionId && + state.client === client && + state.connectionEpoch === connectionEpoch && + resolveUiSelectedSessionAgentId(state) === agentId && + sourceIsCurrent(state, source); + if (!isCurrent()) { + return { admitted: false, message: t("chat.sendErrors.activeLeafChanged") }; + } + if (!validRequest(request)) { + return { admitted: false, message: t("chat.documentViewer.requestUnavailable") }; + } + const attachment = request.cropDataUrl + ? chatAttachmentFromDataUrl( + request.cropDataUrl, + `document-selection-${source.pageUnit}-${request.pageNumber}.png`, + ) + : null; + if (request.cropDataUrl && !attachment) { + return { admitted: false, message: t("chat.documentViewer.requestUnavailable") }; + } + const attachments = attachment ? [attachment] : []; + const filename = truncateUtf16Safe(source.label.split(/[\\/]/u).at(-1) ?? "document", 256); + const page = t( + source.pageUnit === "slide" + ? "chat.documentViewer.slideThumbnail" + : "chat.documentViewer.pageThumbnail", + { page: String(request.pageNumber) }, + ); + const parts = [ + t("chat.documentViewer.changeRequestTitle", { filename, page }), + request.instruction.trim(), + ]; + if (request.selectedText?.trim()) { + const quoted = request.selectedText + .trim() + .split(/\r?\n/u) + .map((line) => `> ${line}`) + .join("\n"); + parts.push(`${t("chat.documentViewer.selectedText")}:\n${quoted}`); + } + if (request.region) { + const percent = (value: number) => String(Math.round(value * 10_000) / 100); + parts.push( + t("chat.documentViewer.changeRequestRegion", { + x: percent(request.region.x), + y: percent(request.region.y), + width: percent(request.region.width), + height: percent(request.region.height), + }), + ); + } + if (attachment) { + parts.push(t("chat.documentViewer.changeRequestCrop")); + } + const text = parts.join("\n\n"); + let admitted = false; + try { + admitted = await new Promise((resolve) => { + void state + .handleSendChat(text, { + literalMessage: true, + ignoreReplyTarget: true, + replyToId: source.messageId, + attachments, + isSubmissionCurrent: isCurrent, + onAdmitted: () => { + admitted = true; + resolve(true); + }, + }) + .then( + () => resolve(false), + () => resolve(false), + ); + }); + return admitted + ? { admitted: true } + : { + admitted: false, + message: t( + isCurrent() + ? "chat.documentViewer.requestUnavailable" + : "chat.sendErrors.activeLeafChanged", + ), + }; + } finally { + if (!admitted) { + releaseChatAttachmentPayloads(attachments); + } + } + }; +} diff --git a/ui/src/pdf-assets.d.ts b/ui/src/pdf-assets.d.ts new file mode 100644 index 0000000000..bd1f2230a5 --- /dev/null +++ b/ui/src/pdf-assets.d.ts @@ -0,0 +1,13 @@ +declare module "virtual:openclaw-pdf-assets" { + export const pdfAssetUrls: { + readonly cMapUrl: string; + readonly standardFontDataUrl: string; + readonly wasmUrl: string; + readonly iccUrl: string; + }; +} + +declare module "pdfjs-dist/build/pdf.worker.min.mjs?worker&url" { + const workerUrl: string; + export default workerUrl; +} diff --git a/ui/src/platformclaw/i18n.test.ts b/ui/src/platformclaw/i18n.test.ts index a9e8a4ae05..639604eeac 100644 --- a/ui/src/platformclaw/i18n.test.ts +++ b/ui/src/platformclaw/i18n.test.ts @@ -57,6 +57,11 @@ describe("PlatformClaw product translations", () => { expect(t("chat.yieldFailed")).toBe("작업을 맡기지 못했습니다"); expect(t("chat.runControls.sendMessage")).toBe("메시지 보내기"); expect(t("chat.runControls.stopGenerating")).toBe("생성 중지"); + expect(t("chat.documentViewer.contents")).toBe("목차"); + expect(t("chat.documentViewer.requestChanges")).toBe("수정 요청"); + expect(t("chat.documentViewer.slidePosition", { current: "2", total: "3" })).toBe( + "전체 3장 중 2번 슬라이드", + ); expect(t("execApproval.expiresIn", { time: "01:30" })).toBe("01:30 후 만료"); expect(t("execApproval.reviewRequest", { agent: "도우미" })).toBe( "도우미의 승인 요청 검토: {command}", @@ -89,6 +94,8 @@ describe("PlatformClaw product translations", () => { expect(t("nav.settingsGroupAdministration")).toBe("Administration"); expect(t("execApproval.allowOnce")).toBe("Allow once"); expect(t("chat.runControls.sendMessage")).toBe("Send message"); + expect(t("chat.documentViewer.contents")).toBe("Contents"); + expect(t("chat.documentViewer.requestChanges")).toBe("Request changes"); expect(t("chat.waitingOnSubagents")).toBe("Waiting on subagents"); expect(t("chat.yieldWaiting")).toBe("Handed off and waiting"); expect(t("chat.yieldResumed")).toBe("Resumed"); diff --git a/ui/src/platformclaw/locales/ko-document-viewer.ts b/ui/src/platformclaw/locales/ko-document-viewer.ts new file mode 100644 index 0000000000..7d1551e2a5 --- /dev/null +++ b/ui/src/platformclaw/locales/ko-document-viewer.ts @@ -0,0 +1,44 @@ +import type { TranslationMap } from "../../i18n/index.ts"; + +export const documentViewerKoreanTranslations = { + pageUnit: "페이지", + slideUnit: "슬라이드", + previewOptions: "미리보기 옵션", + instructionPlaceholder: "예: 제목 바꿔줘", + previousPage: "이전 페이지", + nextPage: "다음 페이지", + pagePosition: "전체 {total}페이지 중 {current}페이지", + previousSlide: "이전 슬라이드", + nextSlide: "다음 슬라이드", + slidePosition: "전체 {total}장 중 {current}번 슬라이드", + slideLabel: "{title}, 전체 {total}장 중 {current}번 슬라이드", + slideThumbnail: "슬라이드 {page}", + slideTextAlternative: "슬라이드 텍스트", + contents: "목차", + pageThumbnail: "페이지 {page}", + previousContents: "이전 페이지 목록", + nextContents: "다음 페이지 목록", + thumbnailError: "썸네일을 불러올 수 없습니다", + requestChanges: "수정 요청", + requestHint: "수정할 부분을 클릭하거나 드래그해서 영역을 선택하세요.", + selectedRegion: "선택한 영역", + selectedText: "선택한 텍스트", + instruction: "수정 요청 내용", + submitRequest: "요청 보내기", + cancelRequest: "요청 취소", + requestUnavailable: "요청을 보내지 못했습니다. 입력한 내용을 유지한 채 다시 시도하세요.", + requestSent: "요청을 보냈습니다", + changeRequestTitle: "{filename} 수정 요청 — {page}", + changeRequestRegion: "선택한 영역: 왼쪽 {x}%, 위쪽 {y}%, 너비 {width}%, 높이 {height}%.", + changeRequestCrop: + "첨부 이미지는 선택한 영역입니다. 원본 문서는 답장의 참조로 연결되며 다시 첨부되지 않습니다.", + requestCropError: "선택한 영역을 캡처하지 못했습니다. 더 작은 영역을 선택해 다시 시도하세요.", + fitPage: "페이지에 맞춤", + fitWidth: "너비에 맞춤", + expand: "문서 미리보기 확대", + collapse: "문서 미리보기 축소", + pageLabel: "{title}, 전체 {total}페이지 중 {current}페이지", + loadError: "문서를 표시할 수 없습니다. 미리보기를 닫았다 다시 열거나 원본 파일을 다운로드하세요.", + passwordProtected: "암호가 필요한 문서입니다. 원본 파일을 다운로드해 여세요.", + textAlternative: "페이지 텍스트", +} satisfies TranslationMap; diff --git a/ui/src/platformclaw/locales/ko.ts b/ui/src/platformclaw/locales/ko.ts index b8b03d03ea..97938658ed 100644 --- a/ui/src/platformclaw/locales/ko.ts +++ b/ui/src/platformclaw/locales/ko.ts @@ -3,6 +3,7 @@ import { announcementTranslations } from "./ko-announcements.ts"; import { baseballTranslations } from "./ko-baseball.ts"; import { credentialFlowTranslations } from "./ko-credentials.ts"; import { dashboardTranslations } from "./ko-dashboard.ts"; +import { documentViewerKoreanTranslations } from "./ko-document-viewer.ts"; import { executionTranslations } from "./ko-execution.ts"; import { fishingTranslations } from "./ko-fishing.ts"; import { mcpTranslations } from "./ko-mcp.ts"; @@ -57,6 +58,7 @@ export const nativeTranslations = { }, }, chat: { + documentViewer: documentViewerKoreanTranslations, copyError: "오류 정보 복사", errorDetails: "오류 정보 보기", errorShareHelp: "오류 정보를 복사해 관리자에게 보내주세요.", diff --git a/ui/src/styles/chat/layout.css b/ui/src/styles/chat/layout.css index 778a0ea678..97338c2987 100644 --- a/ui/src/styles/chat/layout.css +++ b/ui/src/styles/chat/layout.css @@ -1220,15 +1220,6 @@ openclaw-chat-document-attachment { width: 100%; } -.chat-document-attachment__frame { - display: block; - width: 100%; - height: min(70vh, 720px); - min-height: 320px; - border: 0; - border-radius: 8px; -} - .chat-assistant-attachment-card { display: flex; align-items: center; diff --git a/ui/src/test-helpers/control-ui-e2e.ts b/ui/src/test-helpers/control-ui-e2e.ts index 4e97e18fdf..a194cf3d13 100644 --- a/ui/src/test-helpers/control-ui-e2e.ts +++ b/ui/src/test-helpers/control-ui-e2e.ts @@ -362,6 +362,7 @@ export async function startControlUiE2eServer( const [ { createServer }, { controlUiLocaleModulesPlugin }, + { controlUiPdfAssetsPlugin }, { controlUiBrowserOnlySharedModuleAliases, resolveExternalPackageAliasesForVite, @@ -371,6 +372,7 @@ export async function startControlUiE2eServer( ] = await Promise.all([ import("vite"), import("../../config/control-ui-locales.ts"), + import("../../config/control-ui-pdf-assets.ts"), import("../../vite.config.ts"), ]); const repoRoot = resolveRepoRoot(); @@ -394,7 +396,11 @@ export async function startControlUiE2eServer( ], }, publicDir: path.join(uiRoot, "public"), - plugins: [controlUiLocaleModulesPlugin(), controlUiBrowserOnlySharedModuleAliases()], + plugins: [ + controlUiLocaleModulesPlugin(), + controlUiPdfAssetsPlugin(), + controlUiBrowserOnlySharedModuleAliases(), + ], resolve: { alias: [ { find: "json5", replacement: json5EsmPath }, diff --git a/ui/src/test-helpers/platformclaw-control-ui-e2e.ts b/ui/src/test-helpers/platformclaw-control-ui-e2e.ts new file mode 100644 index 0000000000..d6ec9ad013 --- /dev/null +++ b/ui/src/test-helpers/platformclaw-control-ui-e2e.ts @@ -0,0 +1,137 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Page } from "playwright"; +import { PLATFORMCLAW_WEB_GATEWAY_METHODS } from "../../../packages/platformclaw-control-plane/src/browser-gateway-policy.ts"; +import { + createPlatformClawWebAssetHandler, + PLATFORMCLAW_WEB_APP_PATH, + PLATFORMCLAW_WEB_TERMINAL_ENABLED, +} from "../../../packages/platformclaw-control-plane/src/web-assets.ts"; +import { + buildProductionControlUiE2e, + installMockGateway, + type ControlUiE2eServer, + type ControlUiMockGatewayScenario, +} from "./control-ui-e2e.ts"; + +/** Serve the shipped employee document and its real response headers, without CSP substitution. */ +export async function startPlatformClawControlUiE2eServer( + buildId: string, +): Promise { + const buildDirectory = await mkdtemp(path.join(tmpdir(), "platformclaw-employee-e2e-")); + let assets: ReturnType; + const server = createServer((req, res) => { + void (async () => { + if (!(await assets.handlePublic(req, res)) && !(await assets.handleApplication(req, res))) { + res.writeHead(404).end(); + } + })().catch(() => { + if (res.headersSent) { + res.destroy(); + } else { + res.writeHead(500).end(); + } + }); + }); + const close = async () => { + try { + if (server.listening) { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } + } finally { + await rm(buildDirectory, { recursive: true, force: true }); + } + }; + try { + await buildProductionControlUiE2e(buildDirectory, buildId); + const [{ build }, { default: loginConfig }] = await Promise.all([ + import("vite"), + import("../../vite.platformclaw-login.config.ts"), + ]); + await build({ + ...loginConfig({ outDir: buildDirectory }), + configFile: false, + logLevel: "error", + root: fileURLToPath(new URL("../..", import.meta.url)), + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Employee web server did not expose a TCP port"); + } + const origin = `http://127.0.0.1:${address.port}`; + assets = createPlatformClawWebAssetHandler(buildDirectory, { publicOrigin: origin }); + return { baseUrl: `${origin}${PLATFORMCLAW_WEB_APP_PATH}/`, close }; + } catch (error) { + await close(); + throw error; + } +} + +/** Mock employee identity and Gateway data while retaining the real employee asset owner. */ +export async function installPlatformClawMockGateway( + page: Page, + scenario: ControlUiMockGatewayScenario = {}, +) { + await page.route("**/platformclaw/api/auth/session", (route) => + route.fulfill({ + json: { + authenticated: true, + user: { + accountId: "preview.one", + displayName: "Preview One", + department: "Platform Lab", + globalRole: "member", + }, + agent: { agentId: scenario.defaultAgentId?.trim() || "main", state: "active" }, + session: { idleExpiresAt: Date.now() + 60_000, absoluteExpiresAt: Date.now() + 120_000 }, + }, + }), + ); + await page.addInitScript(() => { + localStorage.setItem("platformclaw.product-tour.v1.completed", "true"); + }); + return installMockGateway(page, { + basePath: PLATFORMCLAW_WEB_APP_PATH, + featureMethods: [...PLATFORMCLAW_WEB_GATEWAY_METHODS], + terminalEnabled: PLATFORMCLAW_WEB_TERMINAL_ENABLED, + ...scenario, + }); +} + +/** Record bounded renderer CSP facts without retaining source URLs or attachment tickets. */ +export async function observePlatformClawPreviewCsp(page: Page): Promise { + const violations: string[] = []; + page.on("console", (message) => { + const prefix = "document-preview-csp "; + if (message.text().startsWith(prefix) && violations.length < 8) { + const diagnostic = message.text().slice(prefix.length, prefix.length + 200); + violations.push(diagnostic); + console.log(prefix + diagnostic); + } + }); + await page.addInitScript(() => { + let reported = 0; + document.addEventListener("securitypolicyviolation", (event) => { + if (reported++ < 8) { + console.log( + "document-preview-csp", + JSON.stringify({ + directive: event.effectiveDirective, + blockedScheme: event.blockedURI.split(":", 1)[0]?.slice(0, 16), + disposition: event.disposition, + }), + ); + } + }); + }); + return violations; +} diff --git a/ui/vite.config.ts b/ui/vite.config.ts index 0a7a06c739..96cbcdba18 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -8,6 +8,7 @@ import { brotliCompressSync, constants as zlibConstants, gzipSync } from "node:z import type { Plugin, UserConfig } from "vite"; import { controlUiCodeSplitting } from "./config/control-ui-chunking.ts"; import { controlUiLocaleModulesPlugin } from "./config/control-ui-locales.ts"; +import { controlUiPdfAssetsPlugin } from "./config/control-ui-pdf-assets.ts"; import { normalizeControlUiBuildInfo } from "./src/build-info-normalizers.ts"; import type { ControlUiBuildInfo } from "./src/build-info.ts"; @@ -42,6 +43,7 @@ const commonJsOptimizeDeps = [ const controlUiPrecompressedAssetExtensions = new Set([ ".css", ".js", + ".mjs", ".json", ".svg", ".txt", @@ -69,7 +71,9 @@ export function createControlUiPrecompressedAssetVariants( }, { fileName: `${fileName}.gz`, - source: gzipSync(body, { level: 9 }), + // Fixed lower compressor memory also improves measured startup and total + // asset transfer sizes; gzip decoding still reproduces the final bytes. + source: gzipSync(body, { level: 9, memLevel: 6 }), }, ]; } @@ -489,6 +493,18 @@ export default function controlUiViteConfig(options: { outDir?: string } = {}): ...resolveTsconfigPathAliasesForVite(), ], }, + worker: { + format: "es", + rolldownOptions: { + // The build reserves this directory for off-main-thread JavaScript so + // performance checks can bound workers without relaxing UI chunk limits. + output: { + entryFileNames: "assets/workers/[name]-[hash].js", + chunkFileNames: "assets/workers/[name]-[hash].js", + assetFileNames: "assets/workers/[name]-[hash].[ext]", + }, + }, + }, build: { outDir: buildOutDir, emptyOutDir: true, @@ -513,6 +529,7 @@ export default function controlUiViteConfig(options: { outDir?: string } = {}): plugins: [ controlUiRenderBlockingBootstrapPlugin(), controlUiLocaleModulesPlugin(), + controlUiPdfAssetsPlugin(), controlUiBrowserOnlySharedModuleAliases(), controlUiPrecompressedAssetsPlugin(buildOutDir), controlUiServiceWorkerBuildIdPlugin(buildInfo.buildId, buildOutDir), diff --git a/ui/vitest.config.ts b/ui/vitest.config.ts index d651a9f1e7..74d55de20e 100644 --- a/ui/vitest.config.ts +++ b/ui/vitest.config.ts @@ -13,6 +13,7 @@ import { } from "../test/vitest/vitest.shared.config.ts"; import { uiIsolatedTestFiles } from "../test/vitest/vitest.ui-isolated-paths.mjs"; import { controlUiLocaleModulesPlugin } from "./config/control-ui-locales.ts"; +import { controlUiPdfAssetsPlugin } from "./config/control-ui-pdf-assets.ts"; const here = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(here, ".."); @@ -137,7 +138,11 @@ export default defineConfig({ ...sharedUiTestConfig, projects: [ defineProject({ - plugins: [nativeNodeTestBuiltinsPlugin(), controlUiLocaleModulesPlugin()], + plugins: [ + nativeNodeTestBuiltinsPlugin(), + controlUiLocaleModulesPlugin(), + controlUiPdfAssetsPlugin(), + ], resolve: { alias: workspaceSourceAliases, }, @@ -157,7 +162,11 @@ export default defineConfig({ }, }), defineProject({ - plugins: [nativeNodeTestBuiltinsPlugin(), controlUiLocaleModulesPlugin()], + plugins: [ + nativeNodeTestBuiltinsPlugin(), + controlUiLocaleModulesPlugin(), + controlUiPdfAssetsPlugin(), + ], resolve: { alias: workspaceSourceAliases, }, @@ -174,7 +183,11 @@ export default defineConfig({ }, }), defineProject({ - plugins: [nativeNodeTestBuiltinsPlugin(), controlUiLocaleModulesPlugin()], + plugins: [ + nativeNodeTestBuiltinsPlugin(), + controlUiLocaleModulesPlugin(), + controlUiPdfAssetsPlugin(), + ], resolve: { alias: workspaceSourceAliases, }, @@ -188,7 +201,7 @@ export default defineConfig({ }, }), defineProject({ - plugins: [controlUiLocaleModulesPlugin()], + plugins: [controlUiLocaleModulesPlugin(), controlUiPdfAssetsPlugin()], resolve: { alias: workspaceSourceAliases, }, diff --git a/ui/vitest.node.config.ts b/ui/vitest.node.config.ts index 0fa0c7eba4..924126736a 100644 --- a/ui/vitest.node.config.ts +++ b/ui/vitest.node.config.ts @@ -4,13 +4,14 @@ import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; import { resolveDefaultVitestPool } from "../test/vitest/vitest.shared.config.ts"; import { controlUiLocaleModulesPlugin } from "./config/control-ui-locales.ts"; +import { controlUiPdfAssetsPlugin } from "./config/control-ui-pdf-assets.ts"; const here = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(here, ".."); // Node-only tests for pure logic (no Playwright/browser dependency). export default defineConfig({ - plugins: [controlUiLocaleModulesPlugin()], + plugins: [controlUiLocaleModulesPlugin(), controlUiPdfAssetsPlugin()], resolve: { alias: [ {