Skip to content

feat(desktop): live command output + agent-produced media (R4) #565

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
physercoe merged 1 commit into from
Aug 16, 2026
Merged

feat(desktop): live command output + agent-produced media (R4) #565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions desktop/src/i18n/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,9 @@ const en: Dict = {
'tx.result': 'Result',
'tx.errorLabel': 'Error',
'tx.denied': 'Denied',
'tx.liveOutput': 'Live output',
'tx.outputClipped': 'earlier output trimmed',
'tx.imageUnavailable': 'Image unavailable',
'tx.thinking': 'Thinking…',
'tx.session': 'Session',
'tx.done': 'Done',
Expand Down Expand Up @@ -2692,6 +2695,9 @@ const zh: Dict = {
'tx.result': '结果',
'tx.errorLabel': '错误',
'tx.denied': '已拒绝',
'tx.liveOutput': '实时输出',
'tx.outputClipped': '已省略较早的输出',
'tx.imageUnavailable': '图片不可用',
'tx.thinking': '思考中…',
'tx.session': '会话',
'tx.done': '完成',
Expand Down
56 changes: 56 additions & 0 deletions desktop/src/styles/partials/05-transcript-boards.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1339,6 +1339,62 @@
border-radius: var(--radius-sm);
background: var(--surface);
}
/* An externalized image being fetched by sha, and one we could not fetch.
Both hold the row's height so the transcript doesn't jump when it lands. */
.ev-image-loading {
display: block;
width: 220px;
aspect-ratio: 16/10;
border-radius: var(--radius-sm);
background: var(--surface-sunken, var(--bg));
border: 1px dashed var(--border);
}
.ev-image-missing {
display: inline-flex;
align-items: center;
padding: var(--spacing-s8);
font-size: var(--font-size-caption);
color: var(--text-muted);
border: 1px dashed var(--border);
border-radius: var(--radius-sm);
}

/* ---- R4 live command output ----
E3 streams a running command's stdout/stderr as tool_call_update partials.
The block is always open (you watch a build, you don't unfold it) and scrolls
inside its own box so a long run can't push the composer off screen. */
.ev-stream {
margin-top: var(--spacing-s4);
}
.ev-stream-head {
display: flex;
align-items: center;
gap: var(--spacing-s6);
font-size: var(--font-size-caption);
color: var(--text-muted);
}
.ev-stream-label {
font-weight: 500;
}
.ev-stream-clip {
font-style: italic;
}
/* The scroller is a flex column-reverse so it stays pinned to the newest line
as output arrives — the tail is what you're waiting on. It wraps a single
REAL child (never an anonymous text item, whose flex behaviour is the shaky
part of this pattern), and that child drops .ev-mono's own cap so there is
exactly one scrollbar. */
.ev-stream-body {
max-height: 260px;
overflow-y: auto;
display: flex;
flex-direction: column-reverse;
}
.ev-stream-text {
margin: 0;
max-height: none;
overflow: visible;
}
.digest pre {
white-space: pre-wrap;
word-break: break-word;
Expand Down
9 changes: 8 additions & 1 deletion desktop/src/surfaces/AgentTranscript.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,14 @@ export function AgentTranscript({ agentId, sessionId }: { agentId: string; sessi
}
if (ev.kind === 'tool_call') {
const id = callToolId(ev.payload);
return <EventCard ev={ev} agentId={live} result={id !== undefined ? resultById.get(id) : undefined} />;
return (
<EventCard
ev={ev}
agentId={live}
result={id !== undefined ? resultById.get(id) : undefined}
update={id !== undefined ? updateById.get(id) : undefined}
/>
);
}
// Quote-into-composer only where the composer lives (live mode).
return <EventCard ev={ev} agentId={live} onQuote={mode === 'live' ? quoteToComposer : undefined} />;
Expand Down
24 changes: 20 additions & 4 deletions desktop/src/ui/AgentCompanion.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import { InlineAttentionCards } from './ApprovalCards';
import { pendingAttentionFor } from './approvalRequest';
import { callToolId, EventCard, toFeedEvent } from './EventCard';
import { isHiddenInFeed } from './feedLens';
import { toolCallUpdateParentId } from './toolGroups';
import { LocalAgentLauncher } from './LocalAgentLauncher';

// Cap per-mention file text so a large file can't blow the message context.
Expand Down Expand Up @@ -262,14 +263,23 @@ export function AgentCompanion({

const feed = useMemo(() => events.map((e, i) => toFeedEvent(e, i)), [events]);

const { resultById, nameById, callIds } = useMemo(() => {
// NOTE: a hand-copy of AgentTranscript's `useToolMaps`. The duplication is
// why this one silently lacked `updateById` (so R4's streamed output had
// nowhere to come from here) — worth collapsing into one shared hook, but
// that refactor is deliberately not folded into this wedge.
const { resultById, updateById, nameById, callIds } = useMemo(() => {
const resultById = new Map<string, Entity>();
const updateById = new Map<string, Entity>();
const nameById = new Map<string, string>();
const callIds = new Set<string>();
for (const ev of feed) {
if (ev.kind === 'tool_result') {
const id = str(ev.payload, 'tool_use_id');
if (id !== undefined) resultById.set(id, ev.payload);
} else if (ev.kind === 'tool_call_update') {
// Latest update wins, matching useToolMaps.
const id = toolCallUpdateParentId(ev.payload);
if (id !== undefined) updateById.set(id, ev.payload);
} else if (ev.kind === 'tool_call') {
const id = callToolId(ev.payload);
if (id !== undefined) {
Expand All @@ -279,7 +289,7 @@ export function AgentCompanion({
}
}
}
return { resultById, nameById, callIds };
return { resultById, updateById, nameById, callIds };
}, [feed]);

// Visible feed: hide noise + fold tool_results that a tool_call already shows.
Expand Down Expand Up @@ -434,13 +444,19 @@ export function AgentCompanion({
<div className="companion-feed scroll">
{visible.map((ev) => {
if (ev.kind === 'tool_call') {
// The two props were swapped here: a tool_call needs its `result`
// (ToolCallBody folds it in) and a tool_result needs `callName` (to
// borrow its tool's name back). Passing each the other's prop meant
// the Companion never folded a result into its call card and
// labelled every standalone result "Result".
const id = callToolId(ev.payload);
return (
<EventCard
key={ev.id}
ev={ev}
agentId={agentId}
callName={id !== undefined ? nameById.get(id) : undefined}
result={id !== undefined ? resultById.get(id) : undefined}
update={id !== undefined ? updateById.get(id) : undefined}
/>
);
}
Expand All @@ -451,7 +467,7 @@ export function AgentCompanion({
key={ev.id}
ev={ev}
agentId={agentId}
result={id !== undefined ? resultById.get(id) : undefined}
callName={id !== undefined ? nameById.get(id) : undefined}
/>
);
}
Expand Down
105 changes: 72 additions & 33 deletions desktop/src/ui/EventCard.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { useT, type TLookup } from '../i18n';
import { arr, bool, num, obj, str, type Entity } from '../hub/types';
import { callToolId } from './toolGroups';
import { contextDivider, fmtCost, fmtDuration, turnFooter, type ContextDivider } from './turnMarkers.ts';
import { EventImage, EventImages, StreamedOutput } from './EventMedia';
import { imageRefsOf, mediaRefFrom, resultTextOf, streamedOutputOf } from './toolMedia';

// Re-exported from its new home in toolGroups.ts (the P1 tool-lineage
// substrate) so existing importers keep working.
Expand Down Expand Up @@ -239,8 +241,29 @@ function ToolResultBody({ result }: { result: Entity }): JSX.Element {
const t = useT();
const isErr = bool(result, 'is_error') === true;
const denied = bool(result, 'denied') === true;
const content = str(result, 'content') ?? jsonText(result['content']);
// A tool that returns a picture (claude reading a PNG, or any bridge tool
// behind E4's relay passthrough) sends image BLOCKS, and the drivers forward
// the content verbatim. Painting them beats the old behaviour, which
// jsonText'd the array and printed a screen of base64 at the director.
const media = imageRefsOf(result['content']);
const label = denied ? t('tx.denied') : isErr ? t('tx.errorLabel') : t('tx.result');
if (media.length > 0) {
// The image IS the result. Any text blocks that came with it still show;
// the raw content deliberately does NOT, because "raw" here is megabytes of
// base64 and <details> keeps its children in the DOM even when closed.
const sidecar = resultTextOf(result['content']);
return (
<div className={`ev-result${isErr ? ' err' : ''}`}>
<EventImages media={media} alt={t('tx.result')} />
{sidecar !== '' && (
<Collapsible label={`${label} · ${firstLine(sidecar)}`} open={isErr}>
<pre className="ev-mono">{sidecar}</pre>
</Collapsible>
)}
</div>
);
}
const content = str(result, 'content') ?? jsonText(result['content']);
return (
<div className={`ev-result${isErr ? ' err' : ''}`}>
<Collapsible label={`${label} · ${firstLine(content)}`} open={isErr}>
Expand All @@ -254,13 +277,20 @@ function ToolResultBody({ result }: { result: Entity }): JSX.Element {
/// Arguments disclosure, and the folded-in tool_result. Exported so the P1
/// tool-group card (ToolGroupCard.tsx) reuses it verbatim as a row's lazy
/// detail.
export function ToolCallBody({ p, result }: { p: Entity; result?: Entity }): JSX.Element {
export function ToolCallBody({ p, result, update }: { p: Entity; result?: Entity; update?: Entity }): JSX.Element {
const t = useT();
const name = str(p, 'name') ?? 'tool';
const input = p['input'];
const hasInput = input !== undefined && input !== null && input !== '';
const meta = toolMeta(name, input);
const errored = result !== undefined && bool(result, 'is_error') === true;
// E3 streams a running command's output as tool_call_update partials, each
// carrying the whole buffer so far; useToolMaps has already folded them
// latest-wins onto this call. Once the tool_result lands it carries the same
// bytes (codex's `aggregatedOutput`, which the E3 probe measured as identical
// to the reassembled deltas), so the live block stands down rather than
// printing the output twice.
const streamed = result === undefined ? streamedOutputOf(update) : '';
return (
<div className="ev-tool">
<div className="ev-tool-head">
Expand All @@ -278,6 +308,7 @@ export function ToolCallBody({ p, result }: { p: Entity; result?: Entity }): JSX
<pre className="ev-mono">{jsonText(input)}</pre>
</Collapsible>
)}
<StreamedOutput text={streamed} />
{result !== undefined && <ToolResultBody result={result} />}
</div>
);
Expand Down Expand Up @@ -385,29 +416,36 @@ function InputTextBody({ p }: { p: Entity }): JSX.Element {
function InputImages({ p }: { p: Entity }): JSX.Element | null {
const images = arr(p, 'images');
if (images.length === 0) return null;
// A `blob:sha256/` ref here is not hypothetical and not rare: the hub
// externalizes every payload string leaf over 64 KiB on ingest
// (payload_externalize.go), and any real screenshot's base64 clears that. It
// used to be skipped, so pasted images simply vanished from the transcript at
// the size where they matter most; EventImage resolves them by sha.
const media = images.flatMap((img) => {
const e = (img !== null && typeof img === 'object' ? img : {}) as Entity;
const ref = mediaRefFrom(str(e, 'mime_type'), str(e, 'data'));
// Keep the per-image filename as alt text — it is the only label a
// screen reader (or a broken image) has for a director's attachment.
return ref === undefined ? [] : [{ ref, alt: str(e, 'filename') ?? 'attachment' }];
});
if (media.length === 0) return null;
return (
<div className="ev-images">
{images.map((img, i) => {
const e = (img !== null && typeof img === 'object' ? img : {}) as Entity;
const mime = str(e, 'mime_type') ?? 'image/png';
const data = str(e, 'data') ?? '';
// A blob:sha256/ ref would mean a future hub externalized the payload;
// rendering that needs the blob resolver, so skip rather than break.
if (data === '' || data.startsWith('blob:')) return null;
return (
<img
key={`${String(i)}-${data.length}`}
className="ev-image"
src={`data:${mime};base64,${data}`}
alt={str(e, 'filename') ?? 'attachment'}
/>
);
})}
{media.map((m, i) => (
<EventImage key={m.ref.source === 'blob' ? `b${m.ref.sha}` : `i${String(i)}-${m.ref.data.length}`} media={m.ref} alt={m.alt} />
))}
</div>
);
}

function bodyFor(ev: FeedEvent, t: TLookup, result?: Entity, callName?: string, agentId?: string): ReactNode {
function bodyFor(
ev: FeedEvent,
t: TLookup,
result?: Entity,
callName?: string,
agentId?: string,
update?: Entity,
): ReactNode {
const p = ev.payload;
switch (ev.kind) {
case 'text': {
Expand Down Expand Up @@ -439,33 +477,30 @@ function bodyFor(ev: FeedEvent, t: TLookup, result?: Entity, callName?: string,
case 'attention_request':
return <AttentionRequestBody p={p} />;
case 'tool_call':
return <ToolCallBody p={p} result={result} />;
return <ToolCallBody p={p} result={result} update={update} />;
case 'tool_call_update': {
// Standalone update — only reachable when the parent is gated or missing
// (feedLens folds the rest into the parent card). Mobile parity
// (_toolCallUpdateBody): tool + status kv line and the first text block
// of the ACP content array as a preview.
const title = str(p, 'title') ?? str(p, 'name') ?? 'tool';
const status = str(p, 'status');
let preview: string | undefined;
for (const b of arr(p, 'content')) {
if (b === null || typeof b !== 'object') continue;
const blk = b as Entity;
if (str(blk, 'type') !== 'content') continue;
const inner = obj(blk, 'content');
if (inner !== undefined && str(inner, 'type') === 'text') {
preview = str(inner, 'text');
break;
}
}
// This card is the ONLY place a parentless update's content is visible,
// so it shows the whole streamed output rather than mobile's one-line
// preview — for a command whose parent tool_call never arrived, the first
// line is rarely the one you need. Images render as images (an ACP update
// can carry an image block just as a tool_result can).
const streamed = streamedOutputOf(p);
const media = imageRefsOf(p['content']);
return (
<div className="ev-tool">
<div className="ev-tool-head">
<Icon name="wrench" size={15} className="ev-tool-ico" />
<span className="ev-tool-verb">{title}</span>
{status !== undefined && <span className="muted small">· {status}</span>}
</div>
{preview !== undefined && preview !== '' && <div className="ev-line muted">{firstLine(preview)}</div>}
<EventImages media={media} alt={title} />
<StreamedOutput text={streamed} />
</div>
);
}
Expand Down Expand Up @@ -648,6 +683,7 @@ export const EventCard = memo(function EventCard({
callName,
agentId,
onQuote,
update,
}: {
ev: FeedEvent;
result?: Entity;
Expand All @@ -659,6 +695,9 @@ export const EventCard = memo(function EventCard({
/// Quote this message into the composer (assistant text only). Omitted where
/// there is no composer to quote into.
onQuote?: (text: string) => void;
/// The latest `tool_call_update` folded onto this `tool_call` (useToolMaps'
/// updateById). Carries E3's streamed command output while the tool runs.
update?: Entity;
}): JSX.Element {
const t = useT();
// #332: three tones (user / error / neutral); boxed only where action lives,
Expand All @@ -678,7 +717,7 @@ export const EventCard = memo(function EventCard({
const text = messageText(ev);
return (
<div className={cls} data-seq={ev.seq}>
<div className="ev-body">{bodyFor(ev, t, result, callName, agentId)}</div>
<div className="ev-body">{bodyFor(ev, t, result, callName, agentId, update)}</div>
{text !== undefined && (ev.ts !== undefined || text.trim() !== '') && (
<div className="ev-meta">
{ev.ts !== undefined && <TimeStamp ts={ev.ts} />}
Expand Down
Loading
Loading