diff --git a/apps/desktop/src/main/__tests__/artifact-preview-registry.test.ts b/apps/desktop/src/main/__tests__/artifact-preview-registry.test.ts index 609376de4a..4ae9aa4b6e 100644 --- a/apps/desktop/src/main/__tests__/artifact-preview-registry.test.ts +++ b/apps/desktop/src/main/__tests__/artifact-preview-registry.test.ts @@ -19,9 +19,11 @@ import { strict as assert } from 'node:assert'; import { describe, it } from 'node:test'; -import type { ArtifactBinaryReadResult } from '@maka/core/artifacts'; import { - IMAGE_PAYLOAD_MAX_BYTES, + ARTIFACT_IMAGE_PREVIEW_MAX_BYTES, + type ArtifactBinaryReadResult, +} from '@maka/core/artifacts'; +import { decideImageReadOutcome, resolvePreviewKind, } from '@maka/ui/artifact-preview-registry'; @@ -36,14 +38,17 @@ describe('artifact preview registry', () => { it('enforces the inclusive metadata size boundary before loading', () => { const base = { name: 'image.png', kind: 'image' as const, mimeType: 'image/png' }; - assert.deepEqual(resolvePreviewKind({ ...base, sizeBytes: IMAGE_PAYLOAD_MAX_BYTES }), { + assert.deepEqual(resolvePreviewKind({ ...base, sizeBytes: ARTIFACT_IMAGE_PREVIEW_MAX_BYTES }), { kind: 'image', reason: 'mime_match', }); - assert.deepEqual(resolvePreviewKind({ ...base, sizeBytes: IMAGE_PAYLOAD_MAX_BYTES + 1 }), { - kind: 'unsupported', - reason: 'oversize', - }); + assert.deepEqual( + resolvePreviewKind({ ...base, sizeBytes: ARTIFACT_IMAGE_PREVIEW_MAX_BYTES + 1 }), + { + kind: 'unsupported', + reason: 'oversize', + }, + ); }); it('routes IPC failures and malformed successful payloads without retaining base64', () => { diff --git a/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts b/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts index b5023bd229..6c452d5922 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-artifacts-ipc-main.test.ts @@ -25,6 +25,51 @@ import { test } from "node:test"; import { registerRuntimeHostArtifactsIpc } from "../runtime-host-artifacts-ipc-main.js"; type Handler = (event: unknown, ...args: any[]) => unknown; +type StreamArtifact = ( + sessionId: string, + artifactId: string, + writeChunk: (chunk: Uint8Array) => Promise, +) => Promise; + +function previewArtifact(overrides: Record = {}): Record { + return { + id: "artifact-1", + sessionId: "session-1", + turnId: "turn-1", + createdAt: 1, + name: "preview.png", + kind: "image", + sizeBytes: 4, + mimeType: "image/png", + status: "live", + ...overrides, + }; +} + +function attachmentReadHandler( + artifact: Record, + streamArtifact: StreamArtifact, +): Handler { + const handlers = new Map(); + registerRuntimeHostArtifactsIpc({ + ipcMain: { + handle: (channel, handler) => handlers.set(channel, handler as Handler), + }, + client: { + hostEpoch: "host-1", + async getArtifact() { + return artifact; + }, + streamArtifact, + } as never, + mainWindowController: {} as never, + sendToRenderer() {}, + showItemInFolder() {}, + }); + const handler = handlers.get("attachments:readBytes"); + assert.ok(handler); + return handler; +} test("Runtime Host Artifact IPC preserves previews and streams complete exports", async () => { const root = await mkdtemp(join(tmpdir(), "maka-host-artifact-ipc-")); @@ -40,8 +85,9 @@ test("Runtime Host Artifact IPC preserves previews and streams complete exports" turnId: "turn-1", createdAt: 1, name: "result.bin", - kind: "file", + kind: "image", sizeBytes: content.byteLength, + mimeType: "image/png", status: "live", } as const; const client = { @@ -91,6 +137,14 @@ test("Runtime Host Artifact IPC preserves previews and streams complete exports" await handlers.get("artifacts:readText")?.({}, "session-1", "artifact-1"), { ok: false, reason: "too_large" }, ); + assert.deepEqual( + await handlers.get("attachments:readBytes")?.({}, "session-1", "artifact-1"), + { + ok: true, + base64: content.toString("base64"), + mimeType: "image/png", + }, + ); assert.deepEqual( await handlers.get("app:saveArtifactAs")?.({}, "session-1", "artifact-1"), { ok: true, saved: "result.bin" }, @@ -110,3 +164,33 @@ test("Runtime Host Artifact IPC preserves previews and streams complete exports" await rm(root, { recursive: true, force: true }); } }); + +test("Attachment byte IPC rejects preview-ineligible metadata before streaming", async () => { + for (const [overrides, reason] of [ + [{ id: "artifact-large", sizeBytes: 2 * 1024 * 1024 + 1 }, "too_large"], + [{ id: "artifact-svg", name: "vector.svg", mimeType: "image/svg+xml" }, "unsupported_mime"], + ] as const) { + let streamCalls = 0; + const read = attachmentReadHandler(previewArtifact(overrides), async () => { + streamCalls += 1; + return 0; + }); + assert.deepEqual(await read({}, "session-1", overrides.id), { ok: false, reason }); + assert.equal(streamCalls, 0); + } +}); + +test("Attachment byte IPC stops a stream that exceeds its preview admission", async () => { + const read = attachmentReadHandler( + previewArtifact({ id: "artifact-drifted" }), + async (_sessionId, _artifactId, writeChunk) => { + await writeChunk(new Uint8Array(2 * 1024 * 1024 + 1)); + return 2 * 1024 * 1024 + 1; + }, + ); + + assert.deepEqual( + await read({}, "session-1", "artifact-drifted"), + { ok: false, reason: "too_large" }, + ); +}); diff --git a/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts b/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts index 5ba6f5ed74..dc8ed70083 100644 --- a/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts +++ b/apps/desktop/src/main/runtime-host-artifacts-ipc-main.ts @@ -21,8 +21,12 @@ import { randomUUID } from "node:crypto"; import { open, mkdir, rename, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; -import { MAX_ATTACHMENT_BYTES } from '@maka/core/attachments'; -import { type ArtifactSaveResult } from '@maka/core/artifacts'; +import { + ARTIFACT_IMAGE_PREVIEW_MAX_BYTES, + normalizeArtifactImagePreviewMime, + resolveArtifactImagePreview, + type ArtifactSaveResult, +} from '@maka/core/artifacts'; import { sanitizeArtifactName } from "@maka/storage/artifact-stores"; import { handleReconnectableRead, @@ -40,6 +44,8 @@ interface RuntimeHostArtifactsIpcDeps { readonly presentationRoot?: string; } +const ATTACHMENT_PREVIEW_LIMIT_EXCEEDED = Symbol("attachment-preview-limit-exceeded"); + export function registerRuntimeHostArtifactsIpc( deps: RuntimeHostArtifactsIpcDeps, ): void { @@ -93,27 +99,42 @@ export function registerRuntimeHostArtifactsIpc( const artifact = await deps.client.getArtifact(sessionId, artifactId); if ( !artifact || - artifact.status === "deleted" || - artifact.sizeBytes > MAX_ATTACHMENT_BYTES + artifact.status === "deleted" ) { return { ok: false as const, reason: "not_found" }; } + const preview = resolveArtifactImagePreview(artifact); + if (preview.kind === "unsupported") { + return { + ok: false as const, + reason: preview.reason === "oversize" ? "too_large" : "unsupported_mime", + }; + } + const mimeType = normalizeArtifactImagePreviewMime(artifact.mimeType, artifact.name); + if (!mimeType) return { ok: false as const, reason: "unsupported_mime" }; const chunks: Buffer[] = []; let received = 0; - await deps.client.streamArtifact(sessionId, artifactId, async (chunk) => { - received += chunk.byteLength; - if (received > MAX_ATTACHMENT_BYTES) { - throw new Error("Attachment exceeds the renderer byte limit"); + try { + await deps.client.streamArtifact(sessionId, artifactId, async (chunk) => { + received += chunk.byteLength; + if (received > ARTIFACT_IMAGE_PREVIEW_MAX_BYTES) { + throw ATTACHMENT_PREVIEW_LIMIT_EXCEEDED; + } + chunks.push(Buffer.from(chunk)); + }); + } catch (error) { + if (error === ATTACHMENT_PREVIEW_LIMIT_EXCEEDED) { + return { ok: false as const, reason: "too_large" }; } - chunks.push(Buffer.from(chunk)); - }); + throw error; + } if (received !== artifact.sizeBytes) { return { ok: false as const, reason: "read_failed" }; } return { ok: true as const, base64: Buffer.concat(chunks, received).toString("base64"), - mimeType: artifact.mimeType ?? "application/octet-stream", + mimeType, }; }, ); diff --git a/apps/desktop/src/preload/bridge-contract.d.ts b/apps/desktop/src/preload/bridge-contract.d.ts index 85c431907e..a639c67fab 100644 --- a/apps/desktop/src/preload/bridge-contract.d.ts +++ b/apps/desktop/src/preload/bridge-contract.d.ts @@ -1344,10 +1344,7 @@ export interface MakaBridge { | { ok: true; base64: string; mimeType: string } | { ok: false; reason: string } >; - readBytes(sessionId: string, relativePath: string): Promise< - | { ok: true; base64: string; mimeType: string } - | { ok: false; reason: string } - >; + readBytes(sessionId: string, artifactId: string): Promise; }; search: { thread( diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index c21d120bff..9f59cfad0a 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -2593,11 +2593,8 @@ const makaBridge = { > { return ipcRenderer.invoke('attachments:previewApproval', approvalId); }, - readBytes(sessionId: string, relativePath: string): Promise< - | { ok: true; base64: string; mimeType: string } - | { ok: false; reason: string } - > { - return invokeSessionRuntimeHost('attachments:readBytes', sessionId, relativePath); + readBytes(sessionId: string, artifactId: string): Promise { + return invokeSessionRuntimeHost('attachments:readBytes', sessionId, artifactId); }, }, search: { diff --git a/apps/desktop/src/renderer/features/workbar/ports.ts b/apps/desktop/src/renderer/features/workbar/ports.ts index 4c1981dbfa..8efcf71ace 100644 --- a/apps/desktop/src/renderer/features/workbar/ports.ts +++ b/apps/desktop/src/renderer/features/workbar/ports.ts @@ -177,6 +177,7 @@ export interface WorkbarInspectorService { } export interface WorkbarAttachmentsService { + readBytes(sessionId: string, artifactId: string): Promise; pickFiles(): Promise< | { ok: true; diff --git a/apps/desktop/src/renderer/features/workbar/testing.ts b/apps/desktop/src/renderer/features/workbar/testing.ts index 532076cbdd..9ecf0928f5 100644 --- a/apps/desktop/src/renderer/features/workbar/testing.ts +++ b/apps/desktop/src/renderer/features/workbar/testing.ts @@ -113,6 +113,7 @@ export function createFakeWorkbarServices( subscribeUsageChanges: noopSubscription, }, attachments: { + readBytes: async () => ({ ok: false, reason: 'not_found' }), pickFiles: async () => ({ ok: false, reason: 'cancelled' }), previewApproval: async () => ({ ok: false, reason: 'not configured' }), }, diff --git a/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx b/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx index 88e2b6b1dc..85ef50fd14 100644 --- a/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx +++ b/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx @@ -303,6 +303,7 @@ export function QuoteCompanionPanel(props: { liveTurn={companion.liveTurn} runningStatus={companion.processing} activeSession={companion.companionSession} + onReadAttachmentBytes={attachments.readBytes} deriveTurnPresentation={deriveTurnPresentation} onTurnFooterAction={(turnId, actionId) => { if (actionId === 'regenerate') { diff --git a/apps/desktop/src/renderer/platform/desktop/create-workbar-services.ts b/apps/desktop/src/renderer/platform/desktop/create-workbar-services.ts index 2863994d66..2a741425c7 100644 --- a/apps/desktop/src/renderer/platform/desktop/create-workbar-services.ts +++ b/apps/desktop/src/renderer/platform/desktop/create-workbar-services.ts @@ -104,6 +104,8 @@ export function createDesktopWorkbarServices( bridge.inspector.subscribeUsageChanges(sessionId, handler), }, attachments: { + readBytes: (sessionId, artifactId) => + bridge.attachments.readBytes(sessionId, artifactId), pickFiles: () => bridge.attachments.pickFiles(), previewApproval: (approvalId) => bridge.attachments.previewApproval(approvalId), diff --git a/docs/astryx-surface-file-inventory.md b/docs/astryx-surface-file-inventory.md index 4961847b92..7271da8145 100644 --- a/docs/astryx-surface-file-inventory.md +++ b/docs/astryx-surface-file-inventory.md @@ -5,7 +5,7 @@ Each row is one on-disk product surface file. Regenerated inventory must stay in Wiki bar: Design Conventions · API Use-the-System · Theming · Container Padding. -**Totals:** 222 files — blocker 0, polish 1, aligned 221. +**Totals:** 223 files — blocker 0, polish 1, aligned 222. ## Exclusions (explicit) @@ -190,6 +190,7 @@ Wiki bar: Design Conventions · API Use-the-System · Theming · Container Paddi | `apps/desktop/src/renderer/workhub-surface.tsx` | other | Button | aligned — uses Astryx (Button) | aligned | | `packages/ui/src/astryx-chat-reasoning.tsx` | shell-chrome-or-panel | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `packages/ui/src/astryx-i18n.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | +| `packages/ui/src/attachment-image.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `packages/ui/src/attachment-kinds.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `packages/ui/src/bot-brand-logo.tsx` | ui-composition | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `packages/ui/src/capability-audit-strip.tsx` | ui-composition | Banner | aligned — uses Astryx (Banner) | aligned | diff --git a/docs/astryx-surface-file-inventory.paths b/docs/astryx-surface-file-inventory.paths index 87af07d30e..0f7026dd52 100644 --- a/docs/astryx-surface-file-inventory.paths +++ b/docs/astryx-surface-file-inventory.paths @@ -162,6 +162,7 @@ apps/desktop/src/renderer/work-board-panel.tsx apps/desktop/src/renderer/workhub-surface.tsx packages/ui/src/astryx-chat-reasoning.tsx packages/ui/src/astryx-i18n.tsx +packages/ui/src/attachment-image.tsx packages/ui/src/attachment-kinds.tsx packages/ui/src/bot-brand-logo.tsx packages/ui/src/capability-audit-strip.tsx diff --git a/packages/core/src/artifacts.ts b/packages/core/src/artifacts.ts index 0b068dcf04..02acd38dad 100644 --- a/packages/core/src/artifacts.ts +++ b/packages/core/src/artifacts.ts @@ -21,6 +21,71 @@ export const ARTIFACT_KINDS = ['file', 'diff', 'html', 'image', 'pdf'] as const; export type ArtifactKind = (typeof ARTIFACT_KINDS)[number]; +/** Maximum encoded image payload admitted to a renderer preview. */ +export const ARTIFACT_IMAGE_PREVIEW_MAX_BYTES = 2 * 1024 * 1024; + +const ARTIFACT_IMAGE_PREVIEW_MIME_BY_EXTENSION: Readonly> = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.avif': 'image/avif', +}; + +const ARTIFACT_IMAGE_PREVIEW_MIMES = new Set( + Object.values(ARTIFACT_IMAGE_PREVIEW_MIME_BY_EXTENSION), +); + +export interface ArtifactImagePreviewInput { + name: string; + kind: ArtifactKind; + mimeType?: string; + sizeBytes?: number; +} + +export type ArtifactImagePreviewResolution = + | { kind: 'image'; reason: 'mime_match' | 'ext_fallback' } + | { + kind: 'unsupported'; + reason: 'kind_disallowed' | 'mime_disallowed' | 'no_mime_no_ext' | 'oversize'; + }; + +/** Normalize the raster MIME admitted to renderer image previews. */ +export function normalizeArtifactImagePreviewMime( + mimeType: string | undefined, + name?: string, +): string | null { + if (typeof mimeType === 'string' && mimeType.trim() !== '') { + const normalized = mimeType.trim().toLowerCase(); + return ARTIFACT_IMAGE_PREVIEW_MIMES.has(normalized) ? normalized : null; + } + if (!name) return null; + const dot = name.lastIndexOf('.'); + if (dot <= 0 || dot === name.length - 1) return null; + return ARTIFACT_IMAGE_PREVIEW_MIME_BY_EXTENSION[name.slice(dot).toLowerCase()] ?? null; +} + +/** One metadata policy shared by preview admission and renderer presentation. */ +export function resolveArtifactImagePreview( + input: ArtifactImagePreviewInput, +): ArtifactImagePreviewResolution { + if (input.kind !== 'image') { + return { kind: 'unsupported', reason: 'kind_disallowed' }; + } + if (input.sizeBytes !== undefined && input.sizeBytes > ARTIFACT_IMAGE_PREVIEW_MAX_BYTES) { + return { kind: 'unsupported', reason: 'oversize' }; + } + if (input.mimeType) { + return normalizeArtifactImagePreviewMime(input.mimeType) + ? { kind: 'image', reason: 'mime_match' } + : { kind: 'unsupported', reason: 'mime_disallowed' }; + } + return normalizeArtifactImagePreviewMime(undefined, input.name) + ? { kind: 'image', reason: 'ext_fallback' } + : { kind: 'unsupported', reason: 'no_mime_no_ext' }; +} + export const ARTIFACT_SOURCES = [ 'tool_result', 'tool_result_archive', diff --git a/packages/runtime/src/__tests__/conversation-copy.test.ts b/packages/runtime/src/__tests__/conversation-copy.test.ts index cee3ac1fe1..87223750e7 100644 --- a/packages/runtime/src/__tests__/conversation-copy.test.ts +++ b/packages/runtime/src/__tests__/conversation-copy.test.ts @@ -695,6 +695,33 @@ test('conversation copy rewrites owned references without changing opaque tool p sessionId: 'session-target', relativePath: 'session-target/artifact-target-file.txt', }); + const userMessage = messages[0]; + assert.equal(userMessage?.type, 'user'); + if (userMessage?.type !== 'user') return; + const canonicalAttachment = userMessage.attachments?.[0]; + assert.ok(canonicalAttachment); + const canonical = rewriteConversationCopyMessage( + { + ...userMessage, + attachments: [ + { + ...canonicalAttachment, + ref: { + kind: 'session_file', + sessionId: 'session-source', + relativePath: 'artifact-source', + }, + }, + ], + }, + references, + ); + assert.equal( + canonical.type === 'user' && canonical.attachments?.[0]?.ref.kind === 'session_file' + ? canonical.attachments[0].ref.relativePath + : undefined, + 'artifact-target', + ); assert.deepEqual( rewritten[1]?.type === 'tool_call' ? rewritten[1].args : undefined, messages[1]?.type === 'tool_call' ? messages[1].args : undefined, @@ -1648,12 +1675,20 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi completedAt: 3, }; await runStore.createRun(sourceRun); + const sourceAttachmentText = [ + '![chart](maka://runtime/attachments/artifact-source)', + 'maka://runtime/attachments/artifact-source?session=other', + ].join('\n'); + const targetAttachmentText = [ + '![chart](maka://runtime/attachments/artifact-target)', + 'maka://runtime/attachments/artifact-source?session=other', + ].join('\n'); const sourceEvents: RuntimeEvent[] = [ runtimeEvent({ id: 'event-user', - role: 'user', - author: 'user', - content: { kind: 'text', text: 'hello' }, + role: 'model', + author: 'agent', + content: { kind: 'text', text: sourceAttachmentText }, refs: { artifactId: 'artifact-source' }, }), runtimeEvent({ @@ -1977,6 +2012,14 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi ), ); assert.equal(targetEvents[0]?.refs?.artifactId, 'artifact-target'); + assert.equal( + targetEvents[0]?.content?.kind === 'text' ? targetEvents[0].content.text : undefined, + targetAttachmentText, + ); + assert.equal( + copied.copiedMessages.find((message) => message.type === 'assistant')?.text, + targetAttachmentText, + ); assert.equal(targetEvents[1]?.refs?.sourceInvocationId, 'invocation-target'); assert.deepEqual( targetEvents[1]?.content?.kind === 'function_call' ? targetEvents[1].content.args : undefined, diff --git a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts index b172e83b1b..b0dd3d815d 100644 --- a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts +++ b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts @@ -258,6 +258,39 @@ function equivalentLegacyMessages(): StoredMessage[] { } describe('projectRuntimeEventsToStoredMessages', () => { + test('exposes a session image ref as a Markdown image source to the model', () => { + const replay = buildRuntimeEventModelReplayPlan([ + ev({ + role: 'user', + author: 'user', + content: { + kind: 'text', + text: 'show this', + attachments: [ + { + kind: 'image', + name: 'preview.png', + mimeType: 'image/png', + bytes: 3, + ref: { + kind: 'session_file', + sessionId, + relativePath: 'attachment-123', + }, + }, + ], + }, + }), + ]); + + const item = replay.items[0]; + assert.equal(item?.kind, 'text'); + assert.match( + item?.kind === 'text' ? item.content : '', + /Markdown image source: "maka:\/\/runtime\/attachments\/attachment-123"/, + ); + }); + test('projects user displayText from RuntimeEvent text content', () => { const typed = '/skill:alpha 帮我整理'; const envelope = 'The user explicitly invoked…\n\n\n帮我整理\n'; diff --git a/packages/runtime/src/conversation-copy.ts b/packages/runtime/src/conversation-copy.ts index f688f47ab5..5e1722c44d 100644 --- a/packages/runtime/src/conversation-copy.ts +++ b/packages/runtime/src/conversation-copy.ts @@ -26,6 +26,7 @@ import type { import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; import type { StorageRef, ToolResultContent } from '@maka/core/events'; +import { parseAttachmentResourceRef } from '@maka/core/attachments'; import { markPersisted } from '@maka/core/persisted-value'; import type { StoredMessage } from '@maka/core/session'; import { decodePersistedToolResultContent } from '@maka/core/tool-result-record-schema'; @@ -196,6 +197,12 @@ export function rewriteConversationCopyMessage( message: StoredMessage, references: ConversationCopyMessageReferenceMap, ): StoredMessage { + if (message.type === 'assistant' && references.mode === 'exact') { + return { + ...message, + text: rewriteAttachmentResourceRefs(message.text, references.artifactIds), + }; + } if (message.type === 'user' && message.attachments) { return { ...message, @@ -224,6 +231,17 @@ export function rewriteConversationCopyMessage( return message; } +function rewriteAttachmentResourceRefs( + text: string, + artifactIds: ReadonlyMap, +): string { + return text.replace(/maka:\/\/runtime\/attachments\/[^\s)\]}>`'",;:!]+/g, (candidate) => { + const parsed = parseAttachmentResourceRef(candidate); + const artifactId = parsed ? artifactIds.get(parsed.artifactId) : undefined; + return artifactId ? `maka://runtime/attachments/${artifactId}` : candidate; + }); +} + export async function prepareConversationRuntimeLedgerCopy(input: { readonly sourceSessionId: string; readonly sourceEvents: readonly RuntimeEvent[]; @@ -1071,13 +1089,20 @@ function rewriteRuntimeEventReferences( references: ConversationCopyReferenceMap, ): RuntimeEvent { const content = - event.content?.kind === 'text' && event.content.attachments + event.content?.kind === 'text' ? { ...event.content, - attachments: event.content.attachments.map((attachment) => ({ - ...attachment, - ref: rewriteStorageRef(attachment.ref, references), - })), + ...(references.mode === 'exact' + ? { text: rewriteAttachmentResourceRefs(event.content.text, references.artifactIds) } + : {}), + ...(event.content.attachments + ? { + attachments: event.content.attachments.map((attachment) => ({ + ...attachment, + ref: rewriteStorageRef(attachment.ref, references), + })), + } + : {}), } : event.content?.kind === 'function_response' ? { @@ -1487,6 +1512,14 @@ function rewriteStorageRef( ): StorageRef { if (ref.kind !== 'session_file' || ref.sessionId !== references.sourceSessionId) return ref; if (references.mode === 'preserve_external') return ref; + const artifactId = references.artifactIds.get(ref.relativePath); + if (artifactId) { + return { + ...ref, + sessionId: references.targetSessionId, + relativePath: artifactId, + }; + } const relativePath = references.relativePaths.get(ref.relativePath); if (!relativePath) { throw new Error(`Conversation copy is missing Session file ${ref.relativePath}`); diff --git a/packages/runtime/src/model-history.ts b/packages/runtime/src/model-history.ts index 9793c5bba7..0ead934b5f 100644 --- a/packages/runtime/src/model-history.ts +++ b/packages/runtime/src/model-history.ts @@ -941,6 +941,9 @@ function formatAttachmentRefs(attachments: readonly AttachmentRef[]): string { ? resourceRef ? [ `Read argument: ${JSON.stringify(readArgument)}`, + ...(attachment.kind === 'image' + ? [`Markdown image source: ${JSON.stringify(resourceRef)}`] + : []), 'This is a Session resource, not a workspace file. Use the ref above; never use the display name as a path.', ].join('\n') : `Read argument: ${JSON.stringify(readArgument)}` diff --git a/packages/ui/src/__tests__/attachment-image.test.tsx b/packages/ui/src/__tests__/attachment-image.test.tsx new file mode 100644 index 0000000000..d134ef8312 --- /dev/null +++ b/packages/ui/src/__tests__/attachment-image.test.tsx @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { afterEach, test } from 'node:test'; +import { act } from 'react'; +import { createRoot } from 'react-dom/client'; +import { parseHTML } from 'linkedom'; +import { TurnView } from '../chat-turn.js'; +import { + SessionAttachmentProvider, + type ReadAttachmentBytes, +} from '../attachment-image.js'; +import { LocaleProvider } from '../locale-context.js'; +import { MarkdownBody } from '../markdown-body.js'; +import type { TurnViewModel } from '../materialize.js'; + +const originalGlobals = { + document: globalThis.document, + matchMedia: globalThis.matchMedia, + requestAnimationFrame: globalThis.requestAnimationFrame, + cancelAnimationFrame: globalThis.cancelAnimationFrame, + window: globalThis.window, +}; +const originalActEnvironment = (globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; +}).IS_REACT_ACT_ENVIRONMENT; +const mountedRoots: ReturnType[] = []; + +afterEach(async () => { + for (const root of mountedRoots.splice(0)) await act(() => root.unmount()); + Object.assign(globalThis, { + ...originalGlobals, + IS_REACT_ACT_ENVIRONMENT: originalActEnvironment, + }); +}); + +function domRoot() { + const { document, window } = parseHTML('
'); + Object.assign(globalThis, { + document, + window, + matchMedia: () => ({ matches: false, addEventListener() {}, removeEventListener() {} }), + requestAnimationFrame: () => 1, + cancelAnimationFrame() {}, + IS_REACT_ACT_ENVIRONMENT: true, + }); + const container = document.querySelector('#root'); + assert.ok(container); + const root = createRoot(container); + mountedRoots.push(root); + return { container, root }; +} + +async function renderAttachmentMarkdown(text: string, readBytes: ReadAttachmentBytes) { + const { container, root } = domRoot(); + await act(async () => { + root.render( + + + , + ); + }); + return { container, root }; +} + +const TURN_WITH_IMAGE: TurnViewModel = { + turnId: 'turn-1', + status: 'completed', + partialOutputRetained: false, + user: { + id: 'ask', + role: 'user', + text: 'show this', + ts: 1, + attachments: [{ + kind: 'image', + name: 'preview.png', + mimeType: 'image/png', + bytes: 3, + ref: { kind: 'session_file', sessionId: 'session-1', relativePath: 'attachment-123' }, + }], + }, + tools: [], + notes: [], + startedAt: 1, + timeline: [], +}; + +test('renders a user thumbnail admitted by the shared preview policy', async () => { + const { container, root } = domRoot(); + await act(async () => { + root.render( + + ({ ok: true, base64: 'aW1n', mimeType: 'image/png' })} + > + + + , + ); + }); + + const image = container.querySelector('.maka-user-attachment-thumbnail img'); + assert.ok(image); + assert.equal(image.getAttribute('src'), 'data:image/png;base64,aW1n'); +}); + +test('rejects an oversized user thumbnail before reading it', async () => { + const { container, root } = domRoot(); + let reads = 0; + await act(async () => { + root.render( + + { + reads += 1; + return { ok: false, reason: 'not_found' }; + }} + > + + + , + ); + }); + + assert.equal(container.querySelector('.maka-user-attachment-thumbnail img'), null); + assert.equal(reads, 0); +}); + +test('renders a session attachment referenced by assistant Markdown', async () => { + let readRef: { sessionId: string; artifactId: string } | undefined; + const { container } = await renderAttachmentMarkdown( + '![preview](maka://runtime/attachments/attachment-123)', + async (sessionId, artifactId) => { + readRef = { sessionId, artifactId }; + return { ok: true, base64: 'aW1n', mimeType: 'image/png' }; + }, + ); + + const image = container.querySelector('img[alt="preview"]'); + assert.ok(image); + assert.equal(image.getAttribute('src'), 'data:image/png;base64,aW1n'); + assert.deepEqual(readRef, { sessionId: 'session-1', artifactId: 'attachment-123' }); +}); + +test('keeps unreadable assistant attachments as named placeholders', async () => { + const cases: Array<[string, ReadAttachmentBytes]> = [ + ['missing', async () => ({ ok: false, reason: 'not_found' })], + ['document', async () => ({ ok: true, base64: 'cGRm', mimeType: 'application/pdf' })], + [ + 'large', + async () => ({ + ok: true, + base64: 'a'.repeat(3 * 1024 * 1024), + mimeType: 'image/png', + }), + ], + ]; + for (const [name, readBytes] of cases) { + const { container } = await renderAttachmentMarkdown( + `![${name}](maka://runtime/attachments/attachment-${name})`, + readBytes, + ); + assert.equal(container.querySelector('img'), null); + assert.ok(container.textContent.includes(`[${name}]`)); + } +}); + +test('shares one attachment read across repeated Markdown image refs', async () => { + let reads = 0; + const { container } = await renderAttachmentMarkdown( + [ + '![first](maka://runtime/attachments/attachment-123)', + '![second](maka://runtime/attachments/attachment-123)', + ].join('\n\n'), + async () => { + reads += 1; + return { ok: true, base64: 'aW1n', mimeType: 'image/png' }; + }, + ); + + assert.equal(container.querySelectorAll('img').length, 2); + assert.equal(reads, 1); +}); + +test('retries an attachment image after a transient read failure', async () => { + const markdown = '![preview](maka://runtime/attachments/attachment-123)'; + let reads = 0; + const readBytes: ReadAttachmentBytes = async () => { + reads += 1; + return reads === 1 + ? { ok: false, reason: 'read_failed' } + : { ok: true, base64: 'cmVjb3ZlcmVk', mimeType: 'image/png' }; + }; + const { container, root } = await renderAttachmentMarkdown(markdown, readBytes); + assert.equal(container.querySelector('img'), null); + await act(async () => { + root.render( + + + , + ); + }); + + const image = container.querySelector('img[alt="preview"]'); + assert.ok(image); + assert.equal(image.getAttribute('src'), 'data:image/png;base64,cmVjb3ZlcmVk'); + assert.equal(reads, 2); +}); + +test('renders an attachment when a streaming Markdown image becomes complete', async () => { + const { container, root } = domRoot(); + const markdown = '![preview](maka://runtime/attachments/attachment-123)'; + const readBytes: ReadAttachmentBytes = async () => ({ + ok: true, + base64: 'c3RyZWFt', + mimeType: 'image/png', + }); + await act(async () => { + root.render( + + + , + ); + }); + assert.equal(container.querySelector('img'), null); + + await act(async () => { + root.render( + + + , + ); + }); + + const image = container.querySelector('img[alt="preview"]'); + assert.ok(image); + assert.equal(image.getAttribute('src'), 'data:image/png;base64,c3RyZWFt'); +}); diff --git a/packages/ui/src/__tests__/markdown-body.test.ts b/packages/ui/src/__tests__/markdown-body.test.ts index 5ff813f702..c38ec386cd 100644 --- a/packages/ui/src/__tests__/markdown-body.test.ts +++ b/packages/ui/src/__tests__/markdown-body.test.ts @@ -283,6 +283,8 @@ it('never loads non-allowlisted Markdown image sources', () => { '', 'caption ![inline](custom://private-resource)', '', + '![data](data:image/png;base64,aW1n)', + '', '![reference][avatar]', '', '[avatar]: file:///Users/example/private.png', @@ -299,6 +301,8 @@ it('does not treat navigation and communication schemes as image resources', () 'MAKA://auth/login', 'maka://settings/models', 'maka://compose?text=hello', + 'maka://runtime/attachments/attachment-123?session=other', + 'maka://runtime/attachments/not-an-artifact', 'mailto:user@example.com', ]) { const markup = renderToStaticMarkup(createElement(MarkdownBody, { @@ -310,6 +314,16 @@ it('does not treat navigation and communication schemes as image resources', () } }); +it('shows an attachment placeholder when no session reader is installed', () => { + const markup = renderToStaticMarkup(createElement(MarkdownBody, { + text: '![preview](maka://runtime/attachments/attachment-123)', + })); + + assert.match(markup, />\[preview\] { const fence = (index: number) => [ '```mermaid', diff --git a/packages/ui/src/artifact-preview-registry.ts b/packages/ui/src/artifact-preview-registry.ts index bf26062a45..def9198076 100644 --- a/packages/ui/src/artifact-preview-registry.ts +++ b/packages/ui/src/artifact-preview-registry.ts @@ -19,54 +19,27 @@ /** Safe raster-image preview classification for renderer data URLs. */ -import type { ArtifactBinaryReadResult, ArtifactKind } from '@maka/core/artifacts'; +import { + ARTIFACT_IMAGE_PREVIEW_MAX_BYTES, + normalizeArtifactImagePreviewMime, + resolveArtifactImagePreview, + type ArtifactBinaryReadResult, + type ArtifactImagePreviewInput, + type ArtifactImagePreviewResolution, +} from '@maka/core/artifacts'; import type { UiLocale } from '@maka/core/ui-locale'; import { getSharedUiCopy } from './shared-ui-copy.js'; /** Path and ownership fields are intentionally outside this boundary. */ -export interface ArtifactPreviewInput { - name: string; - kind: ArtifactKind; - mimeType?: string; - sizeBytes?: number; -} - +export type ArtifactPreviewInput = ArtifactImagePreviewInput; export type PreviewResolution = - | { - kind: 'image'; - reason: 'mime_match' | 'ext_fallback'; - } - | { - kind: 'unsupported'; - reason: 'kind_disallowed' | 'mime_disallowed' | 'no_mime_no_ext' | 'oversize' | 'read_failed'; - }; - -/** Maximum decoded image payload allowed into renderer state. */ -export const IMAGE_PAYLOAD_MAX_BYTES = 2 * 1024 * 1024; + | ArtifactImagePreviewResolution + | { kind: 'unsupported'; reason: 'read_failed' }; /** Encoded-length cap, including base64 padding. */ -const IMAGE_PAYLOAD_MAX_BASE64_LENGTH = Math.ceil((IMAGE_PAYLOAD_MAX_BYTES * 4) / 3) + 2; - -/** - * MIME allowlist shared by metadata and post-load validation. SVG is - * intentionally absent. - */ -const ALLOWED_IMAGE_MIMES: ReadonlySet = new Set([ - 'image/png', - 'image/jpeg', - 'image/gif', - 'image/webp', - 'image/avif', -]); - -/** Return a normalized allowlisted MIME for constructing an image data URL. */ -function normalizeAllowedImageMime(mimeType: string | undefined): string | null { - if (typeof mimeType !== 'string') return null; - const mime = mimeType.trim().toLowerCase(); - if (mime === '') return null; - return ALLOWED_IMAGE_MIMES.has(mime) ? mime : null; -} +const IMAGE_PAYLOAD_MAX_BASE64_LENGTH = + Math.ceil((ARTIFACT_IMAGE_PREVIEW_MAX_BYTES * 4) / 3) + 2; /** Post-load decision after payload size and sniffed MIME validation. */ export type ImagePostLoadOutcome = @@ -80,7 +53,7 @@ function decideImagePostLoad(input: { if (exceedsImagePayloadCap(input.base64)) { return { kind: 'unsupported', reason: 'oversize' }; } - const safeMime = normalizeAllowedImageMime(input.mimeType); + const safeMime = normalizeArtifactImagePreviewMime(input.mimeType); if (!safeMime) { return { kind: 'unsupported', reason: 'mime_disallowed' }; } @@ -98,37 +71,8 @@ export function decideImageReadOutcome(readResult: ArtifactBinaryReadResult): Im return decideImagePostLoad({ base64: readResult.base64, mimeType: readResult.mimeType }); } -/** Safe extension fallback when MIME metadata is absent. */ -const ALLOWED_IMAGE_EXTS: ReadonlySet = new Set([ - '.png', - '.jpg', - '.jpeg', - '.gif', - '.webp', - '.avif', -]); - export function resolvePreviewKind(input: ArtifactPreviewInput): PreviewResolution { - if (input.kind !== 'image') { - return { kind: 'unsupported', reason: 'kind_disallowed' }; - } - // Reject by metadata before materializing base64. - if (input.sizeBytes !== undefined && input.sizeBytes > IMAGE_PAYLOAD_MAX_BYTES) { - return { kind: 'unsupported', reason: 'oversize' }; - } - // A present MIME is authoritative; extension fallback cannot override it. - if (input.mimeType) { - const mime = input.mimeType.trim().toLowerCase(); - if (ALLOWED_IMAGE_MIMES.has(mime)) { - return { kind: 'image', reason: 'mime_match' }; - } - return { kind: 'unsupported', reason: 'mime_disallowed' }; - } - const ext = lowercaseExt(input.name); - if (ext && ALLOWED_IMAGE_EXTS.has(ext)) { - return { kind: 'image', reason: 'ext_fallback' }; - } - return { kind: 'unsupported', reason: 'no_mime_no_ext' }; + return resolveArtifactImagePreview(input); } /** Enforce the post-load cap using encoded length without decoding. */ @@ -143,10 +87,3 @@ export function formatPreviewSize(sizeBytes: number | undefined, locale: UiLocal if (sizeBytes < 1024 * 1024) return `${(sizeBytes / 1024).toFixed(1)} KB`; return `${(sizeBytes / (1024 * 1024)).toFixed(1)} MB`; } - -function lowercaseExt(name: string): string | null { - if (typeof name !== 'string') return null; - const idx = name.lastIndexOf('.'); - if (idx <= 0 || idx === name.length - 1) return null; - return name.slice(idx).toLowerCase(); -} diff --git a/packages/ui/src/attachment-image.tsx b/packages/ui/src/attachment-image.tsx new file mode 100644 index 0000000000..19b88e9a64 --- /dev/null +++ b/packages/ui/src/attachment-image.tsx @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + createContext, + useContext, + useEffect, + useMemo, + useState, + type ReactNode, +} from 'react'; +import type { ArtifactBinaryReadResult } from '@maka/core/artifacts'; +import { decideImageReadOutcome } from './artifact-preview-registry.js'; + +/** Host capability for reading bytes from the Runtime Host attachment authority. */ +export type ReadAttachmentBytes = ( + sessionId: string, + artifactId: string, +) => Promise; + +type SessionAttachmentContextValue = { + sessionId: string; + loadImage: (sessionId: string, artifactId: string) => Promise; +}; + +const SessionAttachmentContext = createContext(undefined); + +/** Installs the one session-scoped attachment reader used by every transcript image. */ +export function SessionAttachmentProvider(props: { + sessionId: string; + readBytes?: ReadAttachmentBytes; + children: ReactNode; +}) { + const value = useMemo( + () => { + const readBytes = props.readBytes; + if (!readBytes) return undefined; + const pending = new Map>(); + return { + sessionId: props.sessionId, + loadImage(sessionId: string, artifactId: string) { + const key = `${sessionId}\0${artifactId}`; + const existing = pending.get(key); + if (existing) return existing; + const loaded = readBytes(sessionId, artifactId) + .then((result) => { + const outcome = decideImageReadOutcome(result); + return outcome.kind === 'image' + ? `data:${outcome.safeMime};base64,${outcome.base64}` + : undefined; + }) + .catch(() => undefined); + pending.set(key, loaded); + void loaded.finally(() => { + if (pending.get(key) === loaded) pending.delete(key); + }); + return loaded; + }, + }; + }, + [props.readBytes, props.sessionId], + ); + return ( + + {props.children} + + ); +} + +/** Resolve a session attachment to an internal data URL without exposing host globals. */ +export function useAttachmentImageSource(ref: { + artifactId: string; + sessionId?: string; +} | undefined): string | undefined { + const context = useContext(SessionAttachmentContext); + const artifactId = ref?.artifactId; + const sessionId = ref?.sessionId ?? context?.sessionId; + const loadImage = context?.loadImage; + const [src, setSrc] = useState(undefined); + + useEffect(() => { + setSrc(undefined); + if (!artifactId || !sessionId || !loadImage) return; + let cancelled = false; + loadImage(sessionId, artifactId) + .then((loaded) => { + if (!cancelled) setSrc(loaded); + }) + return () => { + cancelled = true; + }; + }, [artifactId, loadImage, sessionId]); + + return src; +} diff --git a/packages/ui/src/chat-turn.tsx b/packages/ui/src/chat-turn.tsx index a0253207c1..53deba2af2 100644 --- a/packages/ui/src/chat-turn.tsx +++ b/packages/ui/src/chat-turn.tsx @@ -71,6 +71,8 @@ import { getConversationCopy } from './conversation-copy.js'; import { AstryxLocaleProvider } from './astryx-i18n.js'; import { InlineReferenceText } from './inline-reference.js'; import { redactSecrets } from './redact.js'; +import { useAttachmentImageSource } from './attachment-image.js'; +import { resolvePreviewKind } from './artifact-preview-registry.js'; export function LocalizedChatMessage({ accessibleLabel, @@ -89,19 +91,6 @@ export function LocalizedChatMessage({ ); } -/** - * Injected host capability that reads a session attachment's bytes. @maka/ui is - * host-agnostic: it never reaches into the desktop preload or any other host - * global. The desktop renderer threads its attachment reader through this prop; - * non-desktop hosts (Storybook, tests, a future web shell) can omit it or supply - * their own reader, - * in which case an image attachment stays in its pending skeleton. - */ -export type ReadAttachmentBytes = ( - sessionId: string, - relativePath: string, -) => Promise<{ ok: true; base64: string; mimeType: string } | { ok: false }>; - function legacySentSkillTokens(text: string) { const values = new Set( [...text.matchAll(new RegExp(SKILL_INVOCATION_TOKEN_SOURCE, 'g'))].map((match) => match[0]), @@ -109,32 +98,27 @@ function legacySentSkillTokens(text: string) { return [...values].map((value) => ({ value, label: value, variant: 'neutral' as const })); } -function AttachmentImage(props: { attachment: AttachmentRef; onReadAttachmentBytes?: ReadAttachmentBytes }) { - const [src, setSrc] = useState(undefined); - const { onReadAttachmentBytes } = props; - useEffect(() => { - if (props.attachment.ref.kind !== 'session_file') return; - // No host reader (non-desktop host, or the capability wasn't wired): leave the - // thumbnail in its pending skeleton rather than reaching into a host global. - if (!onReadAttachmentBytes) return; - let cancelled = false; - onReadAttachmentBytes(props.attachment.ref.sessionId, props.attachment.ref.relativePath) - .then((result) => { - if (cancelled || !result.ok) return; - setSrc(`data:${result.mimeType};base64,${result.base64}`); - }) - .catch(() => {}); - return () => { - cancelled = true; - }; - }, [props.attachment, onReadAttachmentBytes]); +function AttachmentImage(props: { attachment: AttachmentRef }) { + const preview = resolvePreviewKind({ + name: props.attachment.name, + kind: 'image', + mimeType: props.attachment.mimeType, + sizeBytes: props.attachment.bytes, + }); + const ref = preview.kind === 'image' && props.attachment.ref.kind === 'session_file' + ? { + sessionId: props.attachment.ref.sessionId, + artifactId: props.attachment.ref.relativePath, + } + : undefined; + const src = useAttachmentImageSource(ref); if (!src) { return ( ); } @@ -171,7 +155,6 @@ const UserMessageBody = memo(function UserMessageBody(props: { attachments?: readonly AttachmentRef[]; quotes?: readonly QuoteRef[]; inlineReferences?: readonly InlineReference[]; - onReadAttachmentBytes?: ReadAttachmentBytes; /** When set on a user message, show an edit affordance that starts a revision draft. */ onEditUserMessage?: () => void; editDisabled?: boolean; @@ -254,7 +237,6 @@ const UserMessageBody = memo(function UserMessageBody(props: { ))} @@ -277,7 +259,6 @@ const UserMessageBody = memo(function UserMessageBody(props: { export function TransientUserMessage(props: { message: TransientUserMessageProjection; - onReadAttachmentBytes?: ReadAttachmentBytes; }) { const copy = getConversationCopy(useUiLocale()).messages; const message = props.message; @@ -295,7 +276,6 @@ export function TransientUserMessage(props: { attachments={message.attachments} quotes={message.quotes} inlineReferences={message.inlineReferences} - onReadAttachmentBytes={props.onReadAttachmentBytes} /> @@ -455,13 +435,6 @@ export const TurnView = memo(function TurnView(props: { providerRetry?: LiveProviderRetry; initialLiveContent?: ReadonlyMap; }; - /** - * Injected host reader for image attachment bytes. Threaded down to the user - * message's `AttachmentImage` thumbnails; absent on non-desktop hosts, where - * image thumbnails stay in their pending skeleton. Keeps @maka/ui from - * reaching into the desktop preload directly. - */ - onReadAttachmentBytes?: ReadAttachmentBytes; /** * Open a linked subagent child session in the main chat column. Threaded into * linked subagent tool rows; omitted when the host has no navigation. @@ -580,7 +553,6 @@ export const TurnView = memo(function TurnView(props: { attachments={turn.user.attachments} quotes={turn.user.quotes} inlineReferences={turn.user.inlineReferences} - onReadAttachmentBytes={props.onReadAttachmentBytes} onEditUserMessage={ props.onEditUserMessage && !turn.user.hostOrigin ? () => props.onEditUserMessage?.(turn.turnId) @@ -636,7 +608,6 @@ export const TurnView = memo(function TurnView(props: { attachments={message.attachments} quotes={message.quotes} inlineReferences={message.inlineReferences} - onReadAttachmentBytes={props.onReadAttachmentBytes} /> ); diff --git a/packages/ui/src/chat-view.tsx b/packages/ui/src/chat-view.tsx index 43535b6f75..4ed428d35d 100644 --- a/packages/ui/src/chat-view.tsx +++ b/packages/ui/src/chat-view.tsx @@ -53,7 +53,6 @@ import { TurnRunningStatus, TurnView, TransientUserMessage, - type ReadAttachmentBytes, type TurnFooterActionMeta, type TurnPresentationDeriver, } from './chat-turn.js'; @@ -64,6 +63,10 @@ import { placeChatConversationItems } from './chat-conversation-items.js'; import { useUiLocale } from './locale-context.js'; import { getConversationCopy } from './conversation-copy.js'; import { SessionContextLayer, type SessionContextGoal } from './session-context-layer.js'; +import { + SessionAttachmentProvider, + type ReadAttachmentBytes, +} from './attachment-image.js'; export interface LiveContentActivationSnapshot { turnId: string; @@ -299,12 +302,9 @@ export function ChatView(props: { }; onRevisionNavigate?: (sessionId: string) => void; /** - * Host reader for image attachment bytes, threaded to each turn's user-message - * thumbnails. The desktop shell passes its preload `attachments.readBytes`; - * non-desktop hosts omit it and image thumbnails stay in their pending - * skeleton. Keeps @maka/ui host-agnostic with no direct host-global access. - * Pass an identity-stable reference so the memoized TurnViews keep skipping - * reconciliation on the hot streaming path. + * Host reader for image attachment bytes. The desktop shell passes its preload + * `attachments.readBytes`; non-desktop hosts may omit it. Keeps @maka/ui + * host-agnostic with no direct host-global access. */ onReadAttachmentBytes?: ReadAttachmentBytes; /** @@ -657,11 +657,15 @@ export function ChatView(props: { ); return ( -
+
{props.returnToLatest ? ( )) : null} @@ -758,7 +761,6 @@ export function ChatView(props: { : undefined} lineageBadges={turnPresentation?.lineageBadgesByTurn[turn.turnId]} onLineageBadgeClick={stableLineageBadgeClick} - onReadAttachmentBytes={props.onReadAttachmentBytes} onOpenLinkedSession={ props.onOpenLinkedSession ? stableOpenLinkedSession : undefined } @@ -801,7 +803,6 @@ export function ChatView(props: { ))} {/* #642 fallback: streaming began before the optimistic user turn @@ -891,7 +892,8 @@ export function ChatView(props: { ) ) : null} -
+
+ ); } diff --git a/packages/ui/src/markdown-body.tsx b/packages/ui/src/markdown-body.tsx index bdc08ae4d0..1687ae12b9 100644 --- a/packages/ui/src/markdown-body.tsx +++ b/packages/ui/src/markdown-body.tsx @@ -47,6 +47,8 @@ import { useUiLocale } from './locale-context.js'; import { getSharedUiCopy } from './shared-ui-copy.js'; import { MermaidDiagram } from './mermaid-diagram.js'; import { prepareMarkdownMath } from './markdown-math.js'; +import { parseAttachmentResourceRef } from '@maka/core/attachments'; +import { useAttachmentImageSource } from './attachment-image.js'; const BASE_MARKDOWN_COMPONENTS = { link: MarkdownLink, @@ -130,10 +132,7 @@ export function MarkdownBody(props: { settledText?: string; density?: 'default' | 'compact'; }) { - const source = neutralizeUnsafeMarkdownImages(props.text); - const settledSource = - props.settledText === undefined ? undefined : neutralizeUnsafeMarkdownImages(props.settledText); - const prepared = prepareMarkdownMath(source, settledSource); + const prepared = prepareMarkdownMath(props.text, props.settledText); const safeText = prepared.text; const budgetedText = props.streaming ? safeText : applyMermaidRenderBudget(safeText); const density = props.density ?? 'default'; @@ -250,94 +249,27 @@ function MarkdownCode(props: { } function MarkdownImage(props: { src: string; alt: string }) { + const attachment = parseAttachmentResourceRef(props.src); + const attachmentSrc = useAttachmentImageSource( + attachment ? { artifactId: attachment.artifactId } : undefined, + ); + if (attachment) { + if (!attachmentSrc) return [{props.alt}]; + return ( + {props.alt} + ); + } if (!isSafeMarkdownImageUrl(props.src)) return [{props.alt}]; - // Astryx calls this component only for images inside a paragraph. The shared - // reset makes bare images block-level, so preserve inline flow for badges and - // sentence-level icons; the reset keeps max-width/height. + // Remote images can be badges or sentence-level icons, so preserve Maka's + // existing inline presentation. Session attachments above are content + // previews and deliberately own a block presentation instead. return {props.alt}; } -/** - * Astryx delegates inline images to `components.image`, but its current - * standalone-image branch renders a native `` directly. Neutralize - * unsafe direct-image syntax before parsing so both branches retain Maka's - * existing closed URL allowlist. The scanner follows Astryx's image grammar - * and leaves fenced/inline code unchanged. - */ -function neutralizeUnsafeMarkdownImages(source: string): string { - let fence: string | null = null; - return source - .split('\n') - .map((line) => { - if (fence) { - if (line.startsWith(fence)) fence = null; - return line; - } - - const fenceMatch = line.match(/^(`{3,}|~{3,})(\w*)/); - if (fenceMatch) { - fence = fenceMatch[1]; - return line; - } - - return neutralizeUnsafeImagesInLine(line); - }) - .join('\n'); -} - -function neutralizeUnsafeImagesInLine(line: string): string { - let output = ''; - let cursor = 0; - - while (cursor < line.length) { - if (line[cursor] === '`') { - const tickCount = line[cursor + 1] === '`' - ? line[cursor + 2] === '`' ? 3 : 2 - : 1; - const delimiter = '`'.repeat(tickCount); - const close = line.indexOf(delimiter, cursor + tickCount); - if (close !== -1) { - const end = close + tickCount; - output += line.slice(cursor, end); - cursor = end; - continue; - } - } - - if (line[cursor] === '!' && line[cursor + 1] === '[') { - const altClose = line.indexOf(']', cursor + 2); - if (altClose !== -1 && line[altClose + 1] === '(') { - const srcStart = altClose + 2; - const srcClose = findClosingParen(line, srcStart); - if (srcClose !== -1) { - const src = line.slice(srcStart, srcClose); - if (isSafeMarkdownImageUrl(src)) { - output += line.slice(cursor, srcClose + 1); - } else { - output += `!\\[${line.slice(cursor + 2, srcClose + 1)}`; - } - cursor = srcClose + 1; - continue; - } - } - } - - output += line[cursor]; - cursor++; - } - - return output; -} - -function findClosingParen(text: string, start: number): number { - let depth = 1; - for (let index = start; index < text.length; index++) { - if (text[index] === '(') depth++; - if (text[index] === ')' && --depth === 0) return index; - } - return -1; -} - function isSafeMarkdownImageUrl(url: string): boolean { try { const protocol = new URL(url).protocol; diff --git a/packages/ui/src/styles.css b/packages/ui/src/styles.css index be419c8c33..f768fa45d2 100644 --- a/packages/ui/src/styles.css +++ b/packages/ui/src/styles.css @@ -461,6 +461,17 @@ color: var(--foreground-secondary); } +/* Astryx hands custom images no inline/block placement or spacing wrapper. + Session artifacts are content previews rather than sentence badges, so they + own a bounded block presentation; compact transcript rhythm resets the + top-level margin above through the shared adjacency rules. */ +.maka-markdown-attachment-image { + display: block; + max-width: 100%; + height: auto; + margin-block: var(--space-3) var(--space-4); +} + /* Markdown code renderers are custom components so Mermaid can be loaded only for settled Mermaid fences. Astryx skips its own spacing wrapper when `components.code` is set, so the custom block sits directly in the document diff --git a/packages/ui/stories/attachment.stories.tsx b/packages/ui/stories/attachment.stories.tsx index f49aca2cb5..f6c687a7e2 100644 --- a/packages/ui/stories/attachment.stories.tsx +++ b/packages/ui/stories/attachment.stories.tsx @@ -36,9 +36,9 @@ const BLUE_PNG = // @maka/ui is host-agnostic: image thumbnails read bytes through the injected // `onReadAttachmentBytes` prop, not a host global. The story supplies a fake // reader that echoes the two solid-color PNGs above. -const mockReadBytes = async (_sessionId: string, relativePath: string) => ({ +const mockReadBytes = async (_sessionId: string, artifactId: string) => ({ ok: true as const, - base64: relativePath.includes('metrics') ? BLUE_PNG : RED_PNG, + base64: artifactId.includes('metrics') ? BLUE_PNG : RED_PNG, mimeType: 'image/png', }); @@ -200,6 +200,30 @@ export const ImageThumbnails: Story = { ), }; +// Real path: the assistant cites a Session attachment ref in Markdown → the +// chat's session-scoped reader resolves it through the same image authority as +// the sent-message thumbnail above. +export const AssistantMarkdownImage: Story = { + render: () => ( + + + + ), +}; + // Real path: send one prompt with every durable reference kind → file tokens sit // above the bubble, while inline Skill/file tokens, quote and image keep their own hierarchy. // The narrow frame verifies wrapping without inventing a second product layout.