diff --git a/docs/guide/07-viewing.md b/docs/guide/07-viewing.md index e81a7787..04e76b5e 100644 --- a/docs/guide/07-viewing.md +++ b/docs/guide/07-viewing.md @@ -269,11 +269,19 @@ automatically, polling about once a second. Reloading the page also reattaches t latest consistent snapshot. Images, text/JSON, audio, and video use generic previews; other file types can be opened in a new tab. -Run canvas cards also show the first available recorded image output. A count -indicates multiple distinct available images; select the card to inspect all outputs. -The canvas uses the verified artifact inventory and never evaluates author code to -produce thumbnails. Plan cards have no artifact previews, including steps marked -`done` by the planner. Updates preserve the canvas camera, selection and workflow navigation. +Run canvas cards also show the first available recorded image output. A step without +one shows the first frame of its first MP4 or WebM video. Resting the pointer on it +plays it muted and looping; click it to keep it playing with sound, or use its play +button. The video plays over the card, follows it as you pan and zoom, and stops when it +ends, on Escape, or when another card covers it; played again, it continues where it +stopped. Its controls keep their size at any +zoom (play, seek, time, mute and full screen, fewer on a small card); with the player +focused, Space or K plays and pauses, the arrow keys seek, M mutes and F goes full screen. A count indicates multiple distinct available images +(or videos); select the card to inspect all outputs. The canvas uses the verified +artifact inventory and never evaluates author code to produce thumbnails; a video's +first frame is read in the browser once its card comes into view. Plan cards have no +artifact previews, including steps marked `done` by the planner. Updates preserve the +canvas camera, selection and workflow navigation. These automatic thumbnails use structured run data. The existing authoring `view` declarations are booleans or HTML template paths; custom templates and structured diff --git a/spec/layout.md b/spec/layout.md index a904d5f7..32b79a72 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -112,8 +112,11 @@ authored cells, rule 10 removes empty tracks and rule 11 sizes them. left. Rules 1–10 read no size, so a preview arriving never changes a cell. Footprints come from -declarations: ports, settings and kind. A step whose declared output ports include an image -reserves its preview area in both views, before any artifact exists. +declarations: ports, settings and kind. A step whose declared output ports include an image or a +video reserves its preview area in both views, before any artifact exists: a 16:9 frame as wide as +the card less its padding, in which the file is fitted whole. A recorded card +previews its first image output, or without one its first MP4 or WebM output, which shows its +first frame until it is played (§5.7). Example: `seed → propose (2 takes) → warm, cool, mono, dusk, night → sheet`, and an island `notes → index`. @@ -255,13 +258,14 @@ including those that leave the deck. | Input | Closed deck | Expanded deck | |---|---|---| | Hover, only where `(hover: hover)` | raise the card under the pointer, and its linked partner; leaving restores the resting card | nothing; everything under the veil is inert | +| Rest on a card's video, only where `(hover: hover)` and motion is not reduced | after a moment it plays over the card, muted and looping, with its controls, until the pointer leaves; pressing it keeps it playing, and a press on its picture turns its sound on | the same | | Hover a bookmark row | raise that member, as hovering its card does | the camera glides to that member's card, which is outlined; leaving keeps the camera there | | Click a bookmark row | select that member (a frame member: its first step), which then rests in front | select it, and the camera glides to its card | | Still click on a card, a back card's visible rim or the badge | expand | select that member | -| Click a labelled control on the front card (`Open workflow`) | does what it says | does what it says | +| Click a labelled control on the front card (`Open workflow`, a video's play control) | does what it says | does what it says | | Double-click | counts as one expand click | on a workflow card: open it | | Click the veil | — | collapse one level; the selection is unchanged; a second deck needs a second click | -| Esc | with focus in the canvas: go back a scope (unchanged) | collapse one level, wherever focus is except in a text field; consumed only then | +| Esc | with focus in the canvas: go back a scope (unchanged) | collapse one level, wherever focus is except in a text field or a playing video; consumed only then | | × | — | collapse one level | | Tab | one stop per deck (the front card); back cards have `tabindex=-1` | each card, in grid order | | Enter or Space | expand | select | @@ -271,6 +275,20 @@ including those that leave the deck. | Scope change | — | collapse at once; keep the pre-expand camera | | A deck inside an expanded card | — | a click expands it one level deeper. Only the innermost level is interactive; the levels around it are its veil, so a click on them collapses one level and selects nothing | +A video plays over its card's preview, following the card as the camera and the deck move it, and +selects nothing. Resting on it plays it muted, without taking focus or replacing a video someone +played or pressed. It stops when it ends, when the pointer leaves one that is only hovered, on Esc +(focus returns to its play control) and as soon as anything covers its card: another card raised +over it, its deck resting with another card in front, or the veil of a deck it is not in; a covered +card's video does not start. Until the +video shows its own frame, the card keeps its first frame; the video fades in over it, and out +again when it stops. Resting on it or playing it again +continues where it stopped, unless it played to its end. Its +controls are the viewer's own and keep their size on screen at any zoom: play, seek, time, mute and +full screen, the time going first and then seek and full screen as the player gets narrow. While it +plays they fade once the pointer rests. With focus in the player, Space or K plays and pauses, the +arrow keys seek 5 seconds, M mutes and F goes full screen. + On collapse, focus returns to the deck's front card. Raising a card never moves the focused element in the document. At most one deck per view is expanded, plus its nested levels. There are no dedicated deck buttons, no subview and no URL that selects a member (§11). diff --git a/web/fx-web/src/canvas.css b/web/fx-web/src/canvas.css index dc482bd8..66a52599 100644 --- a/web/fx-web/src/canvas.css +++ b/web/fx-web/src/canvas.css @@ -49,6 +49,47 @@ @media (prefers-reduced-motion: reduce) { .fx-canvas-shimmer-band { animation: none; } } .fx-canvas-image-error { fill: #71717a; font: 11px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; pointer-events: none; } .fx-canvas-image-count { fill: #a1a1aa; font: 10px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; } +/* A video card (video.ts): its first frame, a play control, and while it plays, one player laid over the frame. */ +.fx-canvas-play { cursor: pointer; outline: none; } +.fx-canvas-play circle { fill: #18181bb3; } +.fx-canvas-play:hover circle, .fx-canvas-play:focus-visible circle { fill: #18181b; } +.fx-canvas-play:focus-visible circle { stroke: #2563eb; stroke-width: 2px; } +.fx-canvas-play svg { color: #fff; pointer-events: none; } +.fx-canvas-video.is-unavailable .fx-canvas-play { display: none; } +/* The video fades in over the first frame and play control, which hide once it has; they show again at once as it fades out. */ +.fx-canvas-video .fx-canvas-node-image, .fx-canvas-video .fx-canvas-play { transition: visibility 0s; } +.fx-canvas-video.is-shown .fx-canvas-node-image, .fx-canvas-video.is-shown .fx-canvas-play { visibility: hidden; transition-delay: .15s; } +.fx-canvas-video.is-unavailable .fx-canvas-image-error { visibility: visible; } +.fx-canvas-player { position: absolute; box-sizing: border-box; overflow: hidden; border-radius: 6px; outline: none; container-type: inline-size; font: 11px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; transition: opacity .15s ease; } +.fx-canvas-player:focus-visible { box-shadow: 0 0 0 2px #2563eb; } +/* Until the video shows its frame the card's first frame shows through, and takes the pointer. */ +.fx-canvas-player:not(.is-shown) { opacity: 0; pointer-events: none; } +@starting-style { .fx-canvas-player.is-shown { opacity: 0; } } +@media (prefers-reduced-motion: reduce) { + .fx-canvas-player { transition: none; } + .fx-canvas-video.is-shown .fx-canvas-node-image, .fx-canvas-video.is-shown .fx-canvas-play { transition-delay: 0s; } +} +.fx-canvas-player > video { display: block; width: 100%; height: 100%; object-fit: contain; } +.fx-canvas-player:fullscreen { border-radius: 0; background: #000; } +.fx-canvas-player.is-idle { cursor: none; } +/* Its controls (video-controls.ts) keep their screen size at any zoom; a small player keeps fewer. */ +.fx-video-controls { position: absolute; right: 0; bottom: 0; left: 0; display: flex; align-items: center; gap: 2px; padding: 14px 4px 3px; background: linear-gradient(#0000, #0000008c); color: #fff; transition: opacity .2s; } +.fx-canvas-player.is-idle .fx-video-controls { opacity: 0; } +.fx-video-controls button { display: grid; flex: none; place-items: center; width: 24px; height: 24px; padding: 0; border: 0; border-radius: 5px; background: none; color: inherit; cursor: pointer; } +.fx-video-controls button[hidden] { display: none; } +.fx-video-controls button:hover { background: #ffffff2e; } +.fx-video-controls button:focus-visible { outline: 2px solid #93c5fd; outline-offset: -2px; } +.fx-video-time { flex: none; padding: 0 3px; font-variant-numeric: tabular-nums; white-space: nowrap; } +.fx-video-seek { flex: 1; min-width: 0; height: 16px; margin: 0 4px; background: none; cursor: pointer; appearance: none; } +.fx-video-seek:disabled { cursor: default; } +.fx-video-seek:focus-visible { outline: 2px solid #93c5fd; outline-offset: 1px; border-radius: 2px; } +.fx-video-seek::-webkit-slider-runnable-track { height: 3px; border-radius: 2px; background: linear-gradient(to right, #fff var(--fx-played, 0%), #ffffff80 var(--fx-played, 0%) var(--fx-buffered, 0%), #ffffff40 var(--fx-buffered, 0%)); } +.fx-video-seek::-moz-range-track { height: 3px; border-radius: 2px; background: linear-gradient(to right, #fff var(--fx-played, 0%), #ffffff80 var(--fx-played, 0%) var(--fx-buffered, 0%), #ffffff40 var(--fx-buffered, 0%)); } +.fx-video-seek::-webkit-slider-thumb { width: 10px; height: 10px; margin-top: -3.5px; border: 0; border-radius: 50%; background: #fff; appearance: none; } +.fx-video-seek::-moz-range-thumb { width: 10px; height: 10px; border: 0; border-radius: 50%; background: #fff; } +@container (max-width: 220px) { .fx-video-time { display: none; } } +@container (max-width: 150px) { .fx-video-seek, .fx-video-controls .fx-video-fullscreen { display: none; } .fx-video-controls { justify-content: space-between; } } +@container (max-width: 64px) { .fx-video-controls { display: none; } } .fx-canvas-toolbar { position: absolute; bottom: 16px; left: 16px; display: flex; align-items: center; gap: 1px; padding: 3px; border: 1px solid #e4e4e7; border-radius: 9px; background: #fff; box-shadow: 0 2px 5px #00000008; } .fx-canvas-toolbar button { min-width: 32px; height: 30px; padding: 0 8px; border: 0; border-radius: 5px; background: #fff; color: #3f3f46; font: 13px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; cursor: pointer; } .fx-canvas-toolbar button:hover { background: #f4f4f5; } diff --git a/web/fx-web/src/canvas.ts b/web/fx-web/src/canvas.ts index 4e33acad..3acf56bb 100644 --- a/web/fx-web/src/canvas.ts +++ b/web/fx-web/src/canvas.ts @@ -5,8 +5,19 @@ import { displayPortName } from "./ports"; import { BADGE_BAND, DECK_OFFSET, canvasEdgePath, layoutGraph, type CanvasLayout, type PlacedDeck, type PositionedFrame, type PositionedNode } from "./layout"; import { DECK_BACK, bookmarkLayout, deckBadge, deckStack, deckTicks, expandedColumns, restingFront, tickHolds, type DeckMember } from "./deck"; import { element, nodeCard, runningLabel, short } from "./card"; +import { CardVideos } from "./video"; const EXPANDED_GAP = 32; +/** A card's labelled controls do what they say (§5.7): a press on one neither pans nor opens its deck. */ +const CARD_CONTROLS = ".fx-canvas-open-workflow, .fx-canvas-play"; + +/** + * Whether a key event's target keeps its own keys: a text field, or a card's video player, whose + * Esc closes it. A page's own Esc (closing a panel, a deck) leaves such a target alone. + */ +export function ownsKeys(target: EventTarget | null) { + return Boolean((target as Element | null)?.closest?.("input, textarea, select, [contenteditable], .fx-canvas-player")); +} /** §5.9: expanded decks fit the viewport down to card titles (13 px) about 11 px on screen. */ const LEGIBLE_ZOOM = 11 / 13; @@ -44,6 +55,8 @@ interface Deck { badge: SVGGElement; /** The first member the closed stack shows (§5.3). */ start: number; + /** The member in front, which alone is uncovered while the deck is closed. */ + front: number; } /** A deck card under the pointer; `mark` when it is pointed at from the bookmark. */ type DeckHit = { deck: number; card: number; mark?: boolean }; @@ -84,6 +97,7 @@ export class CanvasController { private readonly deckBar = document.createElement("div"); /** An open deck's bookmark, fixed at the canvas's left edge while its grid moves. */ private readonly scrubber = document.createElement("div"); + private readonly videos: CardVideos; /** The badge's text size on screen last written to `--fx-text`, within its fade. */ private textSize = ""; private layout: CanvasLayout = { nodes: [], edges: [], frames: [], decks: [], width: 0, height: 0 }; @@ -158,20 +172,23 @@ export class CanvasController { const index = this.open ? this.decks.indexOf(this.open.deck) : -1; if (index >= 0) this.markFront(this.decks[index], this.frontOf(index)); }, { signal: this.abort.signal }); - container.replaceChildren(this.svg, this.toolbar, this.empty, this.deckBar, this.scrubber); + this.videos = new CardVideos(container, () => this.canHover.matches && !this.reducedMotion.matches, (element) => this.covered(element)); + container.replaceChildren(this.svg, this.videos.player, this.toolbar, this.empty, this.deckBar, this.scrubber); container.addEventListener("wheel", this.onWheel, { passive: false, signal: this.abort.signal }); this.svg.addEventListener("pointerdown", this.onPointerDown, { signal: this.abort.signal }); this.svg.addEventListener("pointermove", this.onPointerMove, { signal: this.abort.signal }); - this.svg.addEventListener("pointerup", this.onPointerUp, { signal: this.abort.signal }); - this.svg.addEventListener("pointercancel", this.onPointerUp, { signal: this.abort.signal }); - this.svg.addEventListener("pointerleave", () => this.hover(null), { signal: this.abort.signal }); + // A press may end off the SVG: on a video's player that appeared under it, or on the toolbar. + container.addEventListener("pointerup", this.onPointerUp, { signal: this.abort.signal }); + container.addEventListener("pointercancel", this.onPointerUp, { signal: this.abort.signal }); + // The pointer moving onto the player stays on its card. + // Leaving the canvas, not onto a video's player over its card. + container.addEventListener("pointerleave", () => this.hover(null), { signal: this.abort.signal }); // A deck card is the expand target: the capture phase runs before a card's own selection. this.svg.addEventListener("click", this.onDeckClick, { capture: true, signal: this.abort.signal }); // Raising a card moves focus out of the canvas, so Esc listens on the document while open. document.addEventListener("keydown", (event) => { if (event.key !== "Escape" || !this.open || event.defaultPrevented) return; - const target = event.target as HTMLElement | null; - if (target?.closest("input, textarea, select, [contenteditable]")) return; + if (ownsKeys(event.target)) return; event.preventDefault(); this.collapse(); }, { capture: true, signal: this.abort.signal }); @@ -264,6 +281,7 @@ export class CanvasController { } for (const [index, node] of this.layout.nodes.entries()) { const group = nodeCard(node, this.id, index, signal, (scope) => this.onOpenScope(scope)); + if (node.preview?.kind === "video") this.videos.attach(group, node.id, node.preview, signal); group.addEventListener("pointerenter", () => this.highlightConnections(node.id), { signal }); group.addEventListener("pointerleave", () => this.highlightConnections(null), { signal }); group.addEventListener("click", () => this.onSelect(node.id), { signal }); @@ -277,6 +295,7 @@ export class CanvasController { layer(frameOf.get(node.id) ?? null).cards.append(group); this.nodes.set(node.id, group); } + this.videos.redrawn(this.nodes); const items = new Map([...this.layout.nodes, ...frames].map((item) => [item.id, item])); this.decks = this.layout.decks.map((placed, index) => this.deck(placed, index, layer(placed.frame).cards, items)); this.decks.forEach((_, index) => this.restDeck(index)); @@ -359,7 +378,7 @@ export class CanvasController { const names = members.map((member, at) => `${distinct ? titles[at] : member.key ?? titles[at]}${takes && member.take !== null ? ` · take ${member.take}` : ""}`); const bookmark = this.bookmark(placed, index, members, names); parent.append(bookmark); - return { placed, title, parent, cards, boxes, members, badge: group, bookmark, names, start: 0 }; + return { placed, title, parent, cards, boxes, members, badge: group, bookmark, names, start: 0, front: 0 }; } /** @@ -449,6 +468,14 @@ export class CanvasController { this.marked = card; card?.classList.add("is-marked"); } + /** + * Whether something covers this element's card: the veil of the open deck, when the card is + * outside it, or the front card of a closed deck it is in but not in front of. + */ + private covered(element: Element) { + if (this.open && !this.open.deck.cards.some((card) => card.contains(element))) return true; + return this.decks.some((deck) => deck !== this.open?.deck && deck.cards.some((card, at) => at !== deck.front && card.contains(element))); + } /** The deck card that is or holds this node's card, or -1. */ private holding(deck: Deck, id: string) { const node = this.nodes.get(id); @@ -469,6 +496,8 @@ export class CanvasController { for (const at of stack.order) if (stack.shown[at]) deck.parent.insertBefore(deck.cards[at], deck.badge); this.stack(deck, stack.slots); deck.cards.forEach((card, at) => card.classList.toggle("is-buried", !stack.shown[at])); + deck.front = front; + this.videos.recheck(); this.markFront(deck, front); // One tab stop per closed deck: its front card. deck.cards.forEach((card, at) => { if (card.hasAttribute("tabindex")) card.setAttribute("tabindex", at === front ? "0" : "-1"); }); @@ -544,8 +573,7 @@ export class CanvasController { if (target) this.restDeck(target.deck, target.card, !target.mark); } private onDeckClick = (event: MouseEvent) => { - // A labelled control on a card does what it says (§5.7). - if ((event.target as Element).closest(".fx-canvas-open-workflow")) return; + if ((event.target as Element).closest(CARD_CONTROLS)) return; const badge = (event.target as Element).closest(".fx-deck-badge"); const hit = badge ? { deck: Number(badge.dataset.deck), card: 0 } : this.cardAt(event.target); if (!hit || this.open) return; @@ -582,6 +610,7 @@ export class CanvasController { veil.style.transition = `opacity ${duration}ms ease`; veil.classList.add("is-shown"); this.open = { deck, veil, camera, grid }; + this.videos.recheck(); this.showScrubber(index); this.deckBar.replaceChildren(); const heading = document.createElement("span"); @@ -624,7 +653,11 @@ export class CanvasController { for (const card of open.deck.cards) card.style.transition = `transform ${duration}ms ease`; // They slide straight to their places in the stack; settling restacks them as it raises the front. const at = this.decks.indexOf(open.deck); - if (at >= 0) this.stack(open.deck, deckStack(open.deck.members.length, this.frontOf(at)).slots); + if (at >= 0) { + open.deck.front = this.frontOf(at); + this.stack(open.deck, deckStack(open.deck.members.length, open.deck.front).slots); + } + this.videos.recheck(); open.veil.style.transition = `opacity ${duration}ms ease`; open.veil.classList.remove("is-shown"); if (!keepCamera) this.animateCamera(() => this.viewport.restore(open.camera), duration); @@ -740,11 +773,14 @@ export class CanvasController { */ private onPointerDown = (event: PointerEvent) => { const target = event.target as Element; - if (event.button !== 0 || target.closest(".fx-canvas-open-workflow")) return; + if (event.button !== 0 || target.closest(CARD_CONTROLS)) return; this.drag = { pointer: event.pointerId, x: event.clientX, y: event.clientY, startX: event.clientX, startY: event.clientY, moved: false, background: !target.closest("[data-node-id], [data-connection-id], [data-deck], .fx-deck-bookmark"), veil: Boolean(target.closest(".fx-deck-veil")) }; }; private onPointerMove = (event: PointerEvent) => { + // A drag lasts only while its button is held: a release the canvas never saw (outside the + // window) ends it here rather than leaving every later move to pan. + if (this.drag && event.pointerId === this.drag.pointer && (event.buttons & 1) === 0) this.cancelDrag(); if (!this.drag) { this.hover(this.hoverAt(event.target)); return; } if (event.pointerId !== this.drag.pointer) return; if (!this.drag.moved) { @@ -780,6 +816,6 @@ export class CanvasController { dispose() { if (this.viewportFrame !== null) cancelAnimationFrame(this.viewportFrame); this.viewportFrame = null; - this.cancelDrag(); this.abort.abort(); this.graphAbort.abort(); this.resize.disconnect(); this.container.replaceChildren(); + this.cancelDrag(); this.abort.abort(); this.graphAbort.abort(); this.videos.dispose(); this.resize.disconnect(); this.container.replaceChildren(); } } diff --git a/web/fx-web/src/card.ts b/web/fx-web/src/card.ts index 3ce3ac50..e17585a9 100644 --- a/web/fx-web/src/card.ts +++ b/web/fx-web/src/card.ts @@ -40,6 +40,17 @@ export function cardStatus(node: Pick = {}) { + const svg = element("svg", { viewBox: "0 0 24 24", fill: "currentColor", "aria-hidden": "true", ...attributes }); + if (kind === "play") svg.append(element("path", { d: "M7 4.5v15a1 1 0 0 0 1.53.85l12-7.5a1 1 0 0 0 0-1.7l-12-7.5A1 1 0 0 0 7 4.5z" })); + else for (const x of [6, 14]) svg.append(element("rect", { x, y: 5, width: 4, height: 14, rx: 1.25 })); + return svg; +} + /** A state's glyph as a nested SVG square of `size` at (x, y). */ export function stateIconElement(state: string, x: number, y: number, size: number) { const icon = stateIcon(state); @@ -58,10 +69,12 @@ export function short(text: string, limit: number) { return [...text].length > l */ export function nodeCard(node: PositionedNode, canvasId: string, index: number, signal: AbortSignal, onOpen: (scope: string) => void) { const preview = node.preview; + const { settings_y, preview_y, preview_height } = canvasNodeGeometry(node); const portsLabel = node.ports ? `, inputs: ${node.ports.inputs.map((port) => `${displayPortName(port.name)} (${port.type}, ${port.kind})`).join(", ") || "none"}, outputs: ${node.ports.outputs.map((port) => `${displayPortName(port.name)} (${port.type}, ${port.kind})`).join(", ") || "none"}` : node.pending ? "" : ", port metadata not recorded"; const scopeLabel = node.kind === "workflow" ? `, imported workflow, ${node.child_count ?? 0} steps, ${node.failure_count ?? 0} failed` : ""; const state = node.state === "waiting" && node.waiting_for ? `waiting for ${node.waiting_for}` : node.state; - const label = `${node.title}, ${state}${scopeLabel}${portsLabel}${preview ? `, ${preview.label}, ${preview.count} image ${preview.count === 1 ? "output" : "outputs"}` : ""}`; + const count = preview ? `${preview.count} ${preview.kind} ${preview.count === 1 ? "output" : "outputs"}` : ""; + const label = `${node.title}, ${state}${scopeLabel}${portsLabel}${preview ? `, ${preview.label}, ${count}` : ""}`; const group = element("g", { transform: `translate(${node.x},${node.y})`, class: `fx-canvas-node${node.kind === "workflow" ? " fx-canvas-workflow" : node.kind === "boundary" ? " fx-canvas-boundary" : ""}`, role: node.kind === "workflow" ? "group" : "button", tabindex: 0, "data-node-id": node.id, "data-state": node.state, "aria-label": label, ...(node.kind === "workflow" ? { "aria-current": "false" } : { "aria-pressed": "false" }) }); group.append(element("rect", { width: node.width, height: node.height, rx: 11, class: node.pending ? "fx-canvas-node-box fx-canvas-pending" : "fx-canvas-node-box" })); group.append(element("text", { x: 16, y: 27, class: "fx-canvas-node-title" }, short(node.title, node.ports ? 42 : 28))); @@ -87,31 +100,46 @@ export function nodeCard(node: PositionedNode, canvasId: string, index: number, if (!ports.length) group.append(element("text", { x: textX, y: PORT_TOP + 3, "text-anchor": direction === "input" ? "start" : "end", class: "fx-canvas-port-type" }, "None")); } if (node.ports.settings.length) { - const settings = element("text", { x: 16, y: canvasNodeGeometry(node).settings_y + 10, class: "fx-canvas-settings" }, short(`Settings · ${node.ports.settings.map((setting) => setting.name).join(", ")}`, 52)); + const settings = element("text", { x: 16, y: settings_y + 10, class: "fx-canvas-settings" }, short(`Settings · ${node.ports.settings.map((setting) => setting.name).join(", ")}`, 52)); group.append(settings); } } else if (!node.pending && !preview) { group.append(element("text", { x: 16, y: 64, class: "fx-canvas-port-type" }, "Port metadata not recorded")); } + const frame = { x: 16, y: preview_y, width: node.width - 32, height: preview_height }; if (preview) { - const frame = { x: 16, y: canvasNodeGeometry(node).preview_y, width: node.width - 32, height: 140 }; const clipId = `${canvasId}-preview-${index}`; const clip = element("clipPath", { id: clipId }); clip.append(element("rect", { ...frame, rx: 6 })); - group.append(clip, element("rect", { ...frame, rx: 6, fill: `url(#${canvasId}-checker)` })); + group.append(clip); + const backdrop = element("rect", { ...frame, rx: 6, fill: `url(#${canvasId}-checker)` }); + const unavailable = preview.kind === "video" ? "Video unavailable" : "Image unavailable"; const image = element("image", { ...frame, preserveAspectRatio: "xMidYMid meet", "clip-path": `url(#${clipId})`, role: "img", "aria-label": preview.label, class: "fx-canvas-node-image" }); - const fallback = element("text", { x: node.width / 2, y: frame.y + frame.height / 2, "text-anchor": "middle", "dominant-baseline": "middle", class: "fx-canvas-image-error", visibility: "hidden" }, "Image unavailable"); + const fallback = element("text", { x: node.width / 2, y: frame.y + frame.height / 2, "text-anchor": "middle", "dominant-baseline": "middle", class: "fx-canvas-image-error", visibility: "hidden" }, unavailable); image.addEventListener("error", () => { image.setAttribute("visibility", "hidden"); fallback.setAttribute("visibility", "visible"); - group.setAttribute("aria-label", `${node.title}, ${state}${portsLabel}, Image unavailable: ${preview.label}`); + group.setAttribute("aria-label", `${node.title}, ${state}${portsLabel}, ${unavailable}: ${preview.label}`); }, { signal }); - image.setAttribute("href", preview.url); - group.append(image, element("rect", { ...frame, rx: 6, class: "fx-canvas-image-border" }), fallback); - group.append(element("text", { x: node.width - 16, y: node.height - 22, "text-anchor": "end", class: "fx-canvas-image-count" }, `${preview.count} image ${preview.count === 1 ? "output" : "outputs"}`)); + const border = element("rect", { ...frame, rx: 6, class: "fx-canvas-image-border" }); + if (preview.kind === "image") { + image.setAttribute("href", preview.url); + group.append(backdrop, image, border, fallback); + } else { + // A video shows its first frame, which the canvas sets once the card is in view, and a play + // control; the canvas plays it over the frame (video.ts). The backdrop takes the pointer, so + // resting anywhere on the frame plays it muted. + const video = element("g", { class: "fx-canvas-video", "data-digest": preview.digest }); + const play = element("g", { class: "fx-canvas-play", role: "button", tabindex: 0, "aria-label": `Play ${preview.label}` }); + const center = { x: node.width / 2, y: frame.y + frame.height / 2 }; + play.append(element("circle", { cx: center.x, cy: center.y, r: 18 }), + mediaGlyph("play", { x: center.x - 8, y: center.y - 8, width: 16, height: 16 })); + video.append(backdrop, image, border, fallback, play); + group.append(video); + } + group.append(element("text", { x: node.width - 16, y: node.height - 22, "text-anchor": "end", class: "fx-canvas-image-count" }, count)); } else if (canvasPreviewReserved(node)) { - const frame = { x: 16, y: canvasNodeGeometry(node).preview_y, width: node.width - 32, height: 140 }; - // Only a started step shimmers: the declared image may be on its way. Waiting and settled steps keep the empty frame. + // Only a started step shimmers: the declared file may be on its way. Waiting and settled steps keep the empty frame. if (node.state === "running") { const clipId = `${canvasId}-shimmer-${index}`; const clip = element("clipPath", { id: clipId }); diff --git a/web/fx-web/src/export.ts b/web/fx-web/src/export.ts index c6345278..d2b38a66 100644 --- a/web/fx-web/src/export.ts +++ b/web/fx-web/src/export.ts @@ -1,5 +1,5 @@ import { parseViewerRun, type ViewerRun } from "./index"; -import { flatCanvasGraph, parseLayoutReport, type CanvasGraph, type LayoutReport } from "./graph"; +import { cardPreview, flatCanvasGraph, parseLayoutReport, recordedFileDigests, type CanvasGraph, type LayoutReport } from "./graph"; import { projectScopes } from "./scopes"; export interface ExportArtifact { @@ -28,7 +28,6 @@ export interface LoadedRunExport { manifest: RunExport; manifest_url: string } const record = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); const digest = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); const count = (value: unknown): value is number => typeof value === "number" && Number.isSafeInteger(value) && value >= 0; -const imageTypes = new Set(["image/png", "image/jpeg", "image/webp", "image/gif", "image/avif"]); const mediaExtensions: Record = { "image/png": "png", "image/jpeg": "jpg", "image/webp": "webp", "image/gif": "gif", "image/avif": "avif", "video/mp4": "mp4", "video/webm": "webm", "audio/wav": "wav", "audio/mpeg": "mp3", "audio/ogg": "ogg", "audio/flac": "flac", "model/gltf-binary": "glb", @@ -92,14 +91,6 @@ function pendingRepeat(item: unknown) { && (item.waiting_on === undefined || (Array.isArray(item.waiting_on) && item.waiting_on.every((id) => typeof id === "string") && new Set(item.waiting_on).size === item.waiting_on.length)); } -/** Only FX value variants contain artifact references; opaque JSON is never interpreted as wiring. */ -export function exportFileDigests(value: unknown): string[] { - if (!record(value)) return []; - if (record(value.file)) return digest(value.file.digest) ? [value.file.digest] : []; - if (Array.isArray(value.list)) return value.list.flatMap(exportFileDigests); - if (Array.isArray(value.collection)) return value.collection.flatMap((entry) => Array.isArray(entry) && entry.length === 2 ? exportFileDigests(entry[1]) : []); - return []; -} function validateReferences(view: ViewerRun, layout: LayoutReport, artifacts: ExportArtifact[]) { const ids = new Set(view.nodes.map((node) => node.id)); @@ -135,7 +126,7 @@ function validateReferences(view: ViewerRun, layout: LayoutReport, artifacts: Ex } const inventory = new Set(artifacts.map((item) => item.digest)); for (const values of [view.inputs, view.outputs, ...view.nodes.flatMap((node) => [node.with, node.outputs])]) { - for (const value of Object.values(values)) if (exportFileDigests(value).some((id) => !inventory.has(id))) refuse("an artifact reference is missing from the inventory."); + for (const value of Object.values(values)) if (recordedFileDigests(value).some((id) => !inventory.has(id))) refuse("an artifact reference is missing from the inventory."); } const addresses = new Set([ ...view.nodes.map((node) => node.step ?? node.path), @@ -223,7 +214,7 @@ export async function readRunExport(pin: PinnedRunExport, signal?: AbortSignal, return { manifest: parseRunExport(value), manifest_url: url.href }; } -/** Share wiring, scope projection and geometry while resolving images through the static boundary. */ +/** Share wiring, scope projection and geometry while resolving previews through the static boundary. */ export function exportCanvasGraph(loaded: LoadedRunExport, scope: string | null = null): CanvasGraph { const { view, artifacts } = loaded.manifest; const flat = flatCanvasGraph(view); @@ -231,14 +222,12 @@ export function exportCanvasGraph(loaded: LoadedRunExport, scope: string | null for (const node of flat.nodes) { const source = view.nodes.find((item) => item.id === node.source_id); if (!source) continue; - const images = new Map(); - for (const [port, value] of Object.entries(source.outputs)) for (const id of exportFileDigests(value)) { + const preview = cardPreview(source.outputs, (id) => { const artifact = inventory.get(id); - const url = exportArtifactUrl(loaded, id); - if (artifact && imageTypes.has(artifact.media_type) && url && !images.has(id)) images.set(id, { url, label: `${port}: ${artifact.name}` }); - } - const first = images.entries().next().value; - if (first) node.preview = { kind: "image", digest: first[0], ...first[1], count: images.size }; + const url = artifact ? exportArtifactUrl(loaded, id) : null; + return artifact && url ? { media_type: artifact.media_type, name: artifact.name, url } : null; + }); + if (preview) node.preview = preview; } return projectScopes(view, flat, scope); } diff --git a/web/fx-web/src/graph.ts b/web/fx-web/src/graph.ts index 3115be93..00381f9e 100644 --- a/web/fx-web/src/graph.ts +++ b/web/fx-web/src/graph.ts @@ -150,8 +150,11 @@ export function parseLayoutReport(value: unknown): LayoutReport { return value as unknown as LayoutReport; } -export interface CanvasImagePreview { - kind: "image"; digest: string; url: string; label: string; count: number; +/** A card's preview: an image draws as itself, a video as its first frame until it is played. */ +export interface CanvasPreview { + kind: "image" | "video"; digest: string; url: string; label: string; + /** The step's distinct output files of the preview's kind. */ + count: number; } export interface CanvasStep { id: string; source_id: string; title: string; subtitle: string; state: string; pending: boolean; @@ -175,7 +178,7 @@ export interface CanvasStep { child_count?: number; failure_count?: number; errors?: { id: string; title: string; message: string }[]; - preview?: CanvasImagePreview; + preview?: CanvasPreview; ports?: { inputs: CanvasPort[]; outputs: CanvasPort[]; settings: { name: string; type: string }[] }; } export interface CanvasPort { name: string; type: string; kind: "artifact" | "parameter" | "fact" | "value" } @@ -187,13 +190,17 @@ export interface CanvasConnection { export interface CanvasFrame { id: string; title: string; nodes: string[]; parent?: string | null; address?: string; take?: number[]; key?: string; state?: string } export interface CanvasGraph { nodes: CanvasStep[]; edges: CanvasConnection[]; frames?: CanvasFrame[] } -const imageKinds = new Set(["image/png", "image/jpeg", "image/webp", "image/gif", "image/avif"]); +/** The files a card previews: pictures and the videos every browser plays, never active documents. */ +const previewKinds: Record = { + "image/png": "image", "image/jpeg": "image", "image/webp": "image", "image/gif": "image", "image/avif": "image", + "video/mp4": "video", "video/webm": "video", +}; const compareText = (left: string, right: string) => left < right ? -1 : left > right ? 1 : 0; const compareNamed = (left: { name: string; kind?: string }, right: { name: string; kind?: string }) => compareText(left.name, right.name) || compareText(left.kind ?? "", right.kind ?? ""); -/** Follow recorded FX value variants; ordinary JSON values remain opaque. */ -function recordedFileDigests(value: unknown): string[] { +/** Follow recorded FX value variants; ordinary JSON values remain opaque and are never read as wiring. */ +export function recordedFileDigests(value: unknown): string[] { if (!isRecord(value)) return []; if (isRecord(value.file)) return isDigest(value.file.digest) ? [value.file.digest as string] : []; if (Array.isArray(value.list)) return value.list.flatMap(recordedFileDigests); @@ -202,19 +209,28 @@ function recordedFileDigests(value: unknown): string[] { return []; } -/** Select a recorded output image from the host's verified artifact inventory. */ -function imagePreview(node: ViewerRun["nodes"][number], artifacts: Map, apiBase: string): CanvasImagePreview | undefined { - const images = new Map(); - for (const [port, value] of Object.entries(node.outputs)) { - for (const digest of recordedFileDigests(value)) { - const artifact = artifacts.get(digest); - if (artifact?.available && imageKinds.has(artifact.kind) && artifact.url === `${apiBase}/artifacts/${digest}` && !images.has(digest)) { - images.set(digest, { url: artifact.url, label: `${port}: ${artifact.name}` }); - } - } +/** + * A card's preview from a step's output files, in output order: the first image, or without one + * the first video. `resolve` gives a file's type, name and verified URL, or null to leave it out. + */ +export function cardPreview(outputs: Record, resolve: (digest: string) => { media_type: string; name: string; url: string } | null): CanvasPreview | undefined { + const found = { image: new Map(), video: new Map() }; + for (const [port, value] of Object.entries(outputs)) for (const digest of recordedFileDigests(value)) { + const file = resolve(digest); + const kind = file && previewKinds[file.media_type]; + if (file && kind && !found[kind].has(digest)) found[kind].set(digest, { url: file.url, label: `${port}: ${file.name}` }); } - const first = images.entries().next().value; - return first ? { kind: "image", digest: first[0], ...first[1], count: images.size } : undefined; + const kind = found.image.size ? "image" : found.video.size ? "video" : undefined; + const first = kind && found[kind].entries().next().value; + return kind && first ? { kind, digest: first[0], ...first[1], count: found[kind].size } : undefined; +} + +/** Select a recorded output picture or video from the host's verified artifact inventory. */ +function recordedPreview(node: ViewerRun["nodes"][number], artifacts: Map, apiBase: string): CanvasPreview | undefined { + return cardPreview(node.outputs, (digest) => { + const artifact = artifacts.get(digest); + return artifact?.available && artifact.url === `${apiBase}/artifacts/${digest}` ? { media_type: artifact.kind, name: artifact.name, url: artifact.url } : null; + }); } /** Project declared wiring for display; no expressions or execution decisions are evaluated. */ @@ -224,7 +240,7 @@ export function flatCanvasGraph(view: GraphDocument | ViewerRun, apiBase = "/api const artifacts = new Map(plan ? [] : view.artifacts.map((item) => [item.digest, item])); const source = plan ? view.instances : view.nodes; const nodes: CanvasStep[] = source.map((item) => { - const preview = plan ? undefined : imagePreview(item as ViewerRun["nodes"][number], artifacts, apiBase); + const preview = plan ? undefined : recordedPreview(item as ViewerRun["nodes"][number], artifacts, apiBase); const run = plan ? undefined : item as ViewerRun["nodes"][number]; const started_at = run?.started_at; const declared = plan ? view.types?.[item.uses ?? ""]?.ports : (item as ViewerRun["nodes"][number]).ports; diff --git a/web/fx-web/src/layout.ts b/web/fx-web/src/layout.ts index 05c5d517..da85a107 100644 --- a/web/fx-web/src/layout.ts +++ b/web/fx-web/src/layout.ts @@ -13,18 +13,26 @@ const COLUMN_GAP = 100, ROW_GAP = 42, FRAME_PAD = 20, FRAME_TOP = 36, MARGIN = 2 /** Socket rows: the card draws them and edges attach to them. */ export const PORT_TOP = 93, PORT_ROW = 34; -/** §4: a card with a declared image output reserves its preview area before any image exists. */ +/** §4: a card with a declared image or video output reserves its preview area before any file exists. */ export function canvasPreviewReserved(node: CanvasStep) { - return node.preview !== undefined || (node.ports?.outputs.some((port) => port.kind === "artifact" && /^image(?:[/[{?]|$)/.test(port.type)) ?? false); + return node.preview !== undefined || (node.ports?.outputs.some((port) => port.kind === "artifact" && /^(?:image|video)(?:[/[{?]|$)/.test(port.type)) ?? false); } -/** Geometry depends on declarations, never on image loading or measured DOM text. */ +/** A preview frame's height for its width: 16:9, the commonest shape of what steps make. */ +const PREVIEW_ASPECT = 9 / 16; + +/** + * Geometry depends on declarations, never on media loading or measured DOM text. The preview + * frame spans the card less 16 on each side. + */ export function canvasNodeGeometry(node: CanvasStep) { - if (!node.ports) return { width: 250, height: node.preview ? 246 : 106, preview_y: 61, settings_y: 0 }; + const width = node.ports ? 360 : 250; + const preview_height = Math.round((width - 32) * PREVIEW_ASPECT); + if (!node.ports) return { width, height: node.preview ? 61 + preview_height + 45 : 106, preview_y: 61, preview_height, settings_y: 0 }; const rows = Math.max(node.ports.inputs.length, node.ports.outputs.length); const settings_y = 83 + Math.max(1, rows) * PORT_ROW; const preview_y = settings_y + (node.ports.settings.length ? 29 : 5); - return { width: 360, height: preview_y + (canvasPreviewReserved(node) ? 140 + 16 : 0) + (node.kind === "workflow" ? 70 : 42), preview_y, settings_y }; + return { width, height: preview_y + (canvasPreviewReserved(node) ? preview_height + 16 : 0) + (node.kind === "workflow" ? 70 : 42), preview_y, preview_height, settings_y }; } export function canvasNodeSize(node: CanvasStep) { const { width, height } = canvasNodeGeometry(node); diff --git a/web/fx-web/src/video-controls.ts b/web/fx-web/src/video-controls.ts new file mode 100644 index 00000000..d44c9587 --- /dev/null +++ b/web/fx-web/src/video-controls.ts @@ -0,0 +1,183 @@ +import { createElement, Maximize, Minimize, Volume2, VolumeX, type IconNode } from "lucide"; +import { mediaGlyph } from "./card"; + +/** How long the pointer rests before a playing video's controls fade, in milliseconds. */ +const IDLE_DELAY = 2000; +/** How far the arrow keys seek, in seconds. */ +const SEEK_STEP = 5; + +/** A video time as players show it: "0:07", "12:30", "1:02:03". */ +export function videoTime(seconds: number) { + const whole = Number.isFinite(seconds) ? Math.max(0, Math.floor(seconds)) : 0; + const [hours, minutes, rest] = [Math.floor(whole / 3600), Math.floor(whole / 60) % 60, whole % 60]; + const two = (value: number) => String(value).padStart(2, "0"); + return hours ? `${hours}:${two(minutes)}:${two(rest)}` : `${minutes}:${two(rest)}`; +} + +/** Where a seek by `by` seconds lands, within the video. */ +export function seekTarget(at: number, by: number, duration: number) { + const end = Number.isFinite(duration) ? duration : at + Math.max(0, by); + return Math.min(end, Math.max(0, at + by)); +} + +/** What a key does to a focused player; keys on its own buttons and slider stay theirs. */ +export function playerKey(key: string, target: "button" | "slider" | "other"): "toggle" | "back" | "forward" | "mute" | "fullscreen" | "close" | null { + if (key === "Escape") return "close"; + if ((key === " " || key === "Enter") && target === "button") return null; + if ((key === "ArrowLeft" || key === "ArrowRight") && target === "slider") return null; + switch (key.toLowerCase()) { + case " ": case "k": return "toggle"; + case "arrowleft": return "back"; + case "arrowright": return "forward"; + case "m": return "mute"; + case "f": return "fullscreen"; + default: return null; + } +} + +/** + * A video with the viewer's own controls: play, a seek slider, time, mute and full screen, sized in + * screen pixels and fewer when the player is small on screen, so they stay usable at any zoom of + * the canvas under it. While it plays they fade when the pointer rests; a press anywhere shows them. + * `press` sees every press first: it returns true to take a press on the picture for itself. + */ +export class VideoControls { + readonly element = document.createElement("div"); + readonly video = document.createElement("video"); + private readonly bar = document.createElement("div"); + private readonly playButton = this.button("fx-video-play"); + private readonly seek = document.createElement("input"); + private readonly time = document.createElement("span"); + private readonly muteButton = this.button("fx-video-mute"); + private readonly fullscreenButton = this.button("fx-video-fullscreen"); + private idle: number | null = null; + private scrubbing = false; + + constructor(signal: AbortSignal, close: () => void, press: (picture: boolean) => boolean) { + const { element, video, bar, seek } = this; + element.className = "fx-canvas-player"; + element.tabIndex = -1; + element.setAttribute("role", "group"); + element.hidden = true; + video.playsInline = true; + bar.className = "fx-video-controls"; + seek.type = "range"; + seek.className = "fx-video-seek"; + seek.min = "0"; + seek.step = "any"; + seek.setAttribute("aria-label", "Seek"); + this.time.className = "fx-video-time"; + this.fullscreenButton.hidden = !document.fullscreenEnabled && !("webkitEnterFullscreen" in video); + bar.append(this.playButton, seek, this.time, this.muteButton, this.fullscreenButton); + element.append(video, bar); + + const on = (type: K, listener: () => void) => video.addEventListener(type, listener, { signal }); + for (const type of ["play", "pause", "timeupdate", "durationchange", "progress", "volumechange", "emptied"] as const) on(type, () => this.render()); + on("play", () => this.wake()); + on("pause", () => this.wake()); + document.addEventListener("fullscreenchange", () => this.render(), { signal }); + + this.playButton.addEventListener("click", () => this.toggle(), { signal }); + this.muteButton.addEventListener("click", () => { video.muted = !video.muted; }, { signal }); + this.fullscreenButton.addEventListener("click", () => this.fullscreen(), { signal }); + seek.addEventListener("input", () => { video.currentTime = Number(seek.value); this.render(); }, { signal }); + seek.addEventListener("pointerdown", () => { this.scrubbing = true; }, { signal }); + for (const type of ["pointerup", "pointercancel", "change"] as const) seek.addEventListener(type, () => { this.scrubbing = false; }, { signal }); + + // The picture plays and pauses, unless `press` took the press. + let taken = false; + element.addEventListener("pointerdown", (event) => { taken = press(event.target === video); this.wake(); }, { signal }); + video.addEventListener("click", () => { if (taken) taken = false; else this.toggle(); }, { signal }); + element.addEventListener("pointermove", () => this.wake(), { signal }); + element.addEventListener("pointerleave", () => this.rest(), { signal }); + element.addEventListener("focusin", () => this.wake(), { signal }); + element.addEventListener("keydown", (event) => { + const target = event.target === seek ? "slider" : (event.target as Element).closest("button") ? "button" : "other"; + const action = playerKey(event.key, target); + if (!action || event.defaultPrevented || event.metaKey || event.ctrlKey || event.altKey) return; + event.preventDefault(); + event.stopPropagation(); + this.wake(); + if (action === "close") close(); + else if (action === "toggle") this.toggle(); + else if (action === "mute") video.muted = !video.muted; + else if (action === "fullscreen") this.fullscreen(); + else video.currentTime = seekTarget(video.currentTime, action === "back" ? -SEEK_STEP : SEEK_STEP, video.duration); + }, { signal }); + signal.addEventListener("abort", () => this.clearIdle(), { once: true }); + this.render(); + } + + /** Whether the player fills the screen: the canvas then leaves its size to the browser. */ + get fullscreened() { return document.fullscreenElement === this.element; } + + /** Before another video: controls shown, nothing pending. */ + reset() { + this.clearIdle(); + this.scrubbing = false; + this.element.classList.remove("is-idle"); + if (this.fullscreened) void document.exitFullscreen().catch(() => {}); + } + + private toggle() { + if (this.video.paused) void this.video.play().catch(() => {}); + else this.video.pause(); + } + private fullscreen() { + if (this.fullscreened) void document.exitFullscreen().catch(() => {}); + else if (document.fullscreenEnabled) void this.element.requestFullscreen().catch(() => {}); + // iOS shows a video full screen only in its own player. + else (this.video as HTMLVideoElement & { webkitEnterFullscreen?: () => void }).webkitEnterFullscreen?.(); + } + + /** Shows the controls; while the video plays they fade again once the pointer rests. */ + private wake() { + this.clearIdle(); + this.element.classList.remove("is-idle"); + if (!this.video.paused) this.idle = window.setTimeout(() => this.rest(), IDLE_DELAY); + } + private rest() { + this.clearIdle(); + if (!this.video.paused && !this.bar.contains(document.activeElement)) this.element.classList.add("is-idle"); + } + private clearIdle() { + if (this.idle !== null) window.clearTimeout(this.idle); + this.idle = null; + } + + private render() { + const { video, seek } = this; + const duration = Number.isFinite(video.duration) ? video.duration : 0; + const playing = !video.paused; + this.icon(this.playButton, () => mediaGlyph(playing ? "pause" : "play", { width: 15, height: 15 }), playing ? "Pause" : "Play"); + this.icon(this.muteButton, lucide(video.muted ? VolumeX : Volume2), video.muted ? "Unmute" : "Mute"); + this.icon(this.fullscreenButton, lucide(this.fullscreened ? Minimize : Maximize), this.fullscreened ? "Exit full screen" : "Full screen"); + seek.max = String(duration); + seek.disabled = duration === 0; + if (!this.scrubbing) seek.value = String(video.currentTime); + const share = (at: number) => `${duration ? Math.min(100, (at / duration) * 100) : 0}%`; + const buffered = video.buffered.length ? video.buffered.end(video.buffered.length - 1) : 0; + seek.style.setProperty("--fx-played", share(video.currentTime)); + seek.style.setProperty("--fx-buffered", share(buffered)); + const [at, length] = [videoTime(video.currentTime), videoTime(duration)]; + const text = `${at} / ${length}`; + seek.setAttribute("aria-valuetext", `${at} of ${length}`); + if (this.time.textContent !== text) this.time.textContent = text; + } + + private button(className: string) { + const button = document.createElement("button"); + button.type = "button"; + button.className = className; + return button; + } + /** Swaps a button's icon and name only when they change. */ + private icon(button: HTMLButtonElement, icon: () => SVGElement, label: string) { + if (button.getAttribute("aria-label") === label) return; + button.setAttribute("aria-label", label); + button.title = label; + button.replaceChildren(icon()); + } +} + +const lucide = (icon: IconNode) => () => createElement(icon, { width: 15, height: 15, "stroke-width": 2, "aria-hidden": "true" }); diff --git a/web/fx-web/src/video.ts b/web/fx-web/src/video.ts new file mode 100644 index 00000000..208ddbca --- /dev/null +++ b/web/fx-web/src/video.ts @@ -0,0 +1,326 @@ +import type { CanvasPreview } from "./graph"; +import { VideoControls } from "./video-controls"; + +/** The widest first frame kept, in pixels: a card's frame is 328 canvas units wide. */ +const STILL_WIDTH = 640; +/** How many videos load at once for their first frames. */ +const LOADS = 2; +/** How long the pointer rests on a video's frame before it plays there, muted, in milliseconds. */ +const HOVER_DELAY = 300; +/** How long the player fades in and out over the card's first frame, in milliseconds (canvas.css). */ +const FADE = 150; +/** How long a stopped video stays loaded, so playing it again continues at once, in milliseconds. */ +const KEEP_LOADED = 30_000; + +/** Where a video stopped at `time` plays from next: there, or from the start once it has played through. */ +export function resumeAt(time: number, duration: number): number | null { + if (!Number.isFinite(time) || time <= 0) return null; + return Number.isFinite(duration) && time >= duration - 0.25 ? null : time; +} + +/** + * A video's first frame drawn on a canvas, or null when the video does not load or this page may + * not read its pixels: a file on another host must allow this page (CORS), or it does not load. + * The video lets go of its file before the caller encodes the frame. + */ +async function drawFirstFrame(url: string, signal: AbortSignal): Promise { + if (signal.aborted) return null; + const video = document.createElement("video"); + video.crossOrigin = "anonymous"; + video.muted = true; + video.playsInline = true; + video.preload = "auto"; + let settle = (_loaded: boolean) => {}; + const abort = () => settle(false); + signal.addEventListener("abort", abort, { once: true }); + try { + const loaded = await new Promise((resolve) => { + settle = resolve; + video.addEventListener("loadeddata", () => resolve(true), { once: true }); + video.addEventListener("error", () => resolve(false), { once: true }); + video.src = url; + }); + if (!loaded || !video.videoWidth || !video.videoHeight) return null; + const scale = Math.min(1, STILL_WIDTH / video.videoWidth); + const canvas = document.createElement("canvas"); + canvas.width = Math.max(1, Math.round(video.videoWidth * scale)); + canvas.height = Math.max(1, Math.round(video.videoHeight * scale)); + canvas.getContext("2d")?.drawImage(video, 0, 0, canvas.width, canvas.height); + return canvas; + } catch { + return null; + } finally { + signal.removeEventListener("abort", abort); + video.removeAttribute("src"); + video.load(); + } +} + +/** A video's first frame as a picture, or null (drawFirstFrame). */ +async function firstFrame(url: string, signal: AbortSignal): Promise { + const canvas = await drawFirstFrame(url, signal); + if (!canvas) return null; + try { + // WebP keeps transparency; a browser that cannot write it gives PNG. + return await new Promise((resolve) => canvas.toBlob(resolve, "image/webp")); + } catch { + return null; + } finally { + canvas.width = canvas.height = 0; + } +} + +/** The playing video: its step, its file, its card's video group and frame, and whether it only plays while hovered. */ +interface Playing { node: string; preview: CanvasPreview; video: SVGGElement; frame: Element; hover: boolean } + +/** + * Video previews on node cards. SVG has no video that every browser draws in place: Safari draws + * an HTML video inside `foreignObject` outside the SVG's transforms and above the rest of it. So a + * card shows its video's first frame as an ordinary SVG image, read once per file when the card + * first comes into view, and one HTML player with the viewer's own controls (video-controls.ts), + * laid over the card's frame, plays it: with sound from its play control, or muted and looping + * while the pointer rests on the frame. It never plays, or goes on playing, over a covered card: + * the canvas says what covers one. + */ +export class CardVideos { + /** First frames by digest: an object URL, null when the file could not be read, undefined while reading. */ + private readonly stills = new Map(); + /** Files the player could not play: their cards say so and offer no play control. */ + private readonly failed = new Set(); + /** Where each file stopped short of its end: hovering or playing it again continues there. */ + private readonly positions = new Map(); + /** First frames waiting for their card to come into view. */ + private readonly waiting = new Map(); + private readonly observer = new IntersectionObserver((entries) => { + for (const entry of entries) { + const preview = entry.isIntersecting ? this.waiting.get(entry.target) : undefined; + if (!preview) continue; + this.observer.unobserve(entry.target); + this.waiting.delete(entry.target); + this.read(preview); + } + }, { rootMargin: "200px" }); + private readonly queue: { digest: string; url: string }[] = []; + private loading = 0; + private readonly abort = new AbortController(); + private readonly controls: VideoControls; + /** The player: the video and its controls, laid over the playing card's frame. */ + readonly player: HTMLDivElement; + private readonly media: HTMLVideoElement; + private playing: Playing | null = null; + /** The player's last placement, written only when it changes. */ + private box = ""; + // Timer and frame handles; clearing 0 or a spent one does nothing. + private following = 0; + private hoverTimer = 0; + private hideTimer = 0; + private unloadTimer = 0; + + constructor(private readonly host: HTMLElement, private readonly hoverPlays: () => boolean, private readonly covered: (element: Element) => boolean) { + const signal = this.abort.signal; + // Leaving a hovered video stops it. Pressing it keeps it playing, and a press on its picture + // turns its sound on rather than pausing it. + this.controls = new VideoControls(signal, () => this.stop(), (picture) => { + if (!this.playing?.hover) return false; + this.playing.hover = false; + this.media.loop = false; + if (picture) this.media.muted = false; + this.player.focus({ preventScroll: true }); + return picture; + }); + this.player = this.controls.element; + this.media = this.controls.video; + this.media.addEventListener("ended", () => this.stop(), { signal }); + this.media.addEventListener("error", () => { + if (!this.playing || !this.media.getAttribute("src")) return; + this.failed.add(this.playing.preview.digest); + this.playing.video.classList.add("is-unavailable"); + this.stop(); + this.unload(); + }, { signal }); + for (const type of ["loadeddata", "seeked", "playing"] as const) this.media.addEventListener(type, this.settle, { signal }); + // A time set before the video loaded is where it starts; a browser that drops it seeks once it can. + this.media.addEventListener("loadedmetadata", () => { + const at = this.playing && this.positions.get(this.playing.preview.digest); + if (at && this.media.currentTime < at - 0.5) this.media.currentTime = at; + }, { signal }); + this.player.addEventListener("pointerleave", () => { if (this.playing?.hover) this.stop(); }, { signal }); + } + + /** Wires a drawn card's video: its first frame once the card is in view, and its play control. */ + attach(card: SVGGElement, node: string, preview: CanvasPreview, signal: AbortSignal) { + const video = card.querySelector(".fx-canvas-video"); + const image = video?.querySelector("image"); + const frame = video?.querySelector(".fx-canvas-image-border"); + const play = video?.querySelector(".fx-canvas-play"); + if (!video || !image || !frame || !play) return; + if (this.failed.has(preview.digest)) video.classList.add("is-unavailable"); + const still = this.stills.get(preview.digest); + if (still) image.setAttribute("href", still); + // A first frame already being read reaches this card when it is ready (read). + else if (!this.stills.has(preview.digest)) { + this.waiting.set(image, preview); + this.observer.observe(image); + signal.addEventListener("abort", () => { this.observer.unobserve(image); this.waiting.delete(image); }, { once: true }); + } + // A labelled control: it plays, and neither selects the card nor opens its deck. + const start = (event: Event) => { + event.preventDefault(); + event.stopPropagation(); + this.play({ node, preview, video, frame }); + }; + play.addEventListener("click", start, { signal }); + play.addEventListener("keydown", (event) => { if (event.key === "Enter" || event.key === " ") start(event); }, { signal }); + // Resting on the frame plays it muted, unless a video played with sound is on. A press waits + // for its release, so the player never appears under a click. + const rest = () => { + window.clearTimeout(this.hoverTimer); + if (!this.hoverPlays() || this.failed.has(preview.digest)) return; + this.hoverTimer = window.setTimeout(() => { + if (video.isConnected && !(this.playing && (!this.playing.hover || this.playing.video === video))) this.play({ node, preview, video, frame }, true); + }, HOVER_DELAY); + }; + video.addEventListener("pointerenter", rest, { signal }); + video.addEventListener("pointerdown", () => window.clearTimeout(this.hoverTimer), { signal }); + video.addEventListener("pointerup", rest, { signal }); + // The player appears under the pointer: moving onto it stays on the frame. + video.addEventListener("pointerleave", (event) => { + if (this.player.contains(event.relatedTarget as Node | null)) return; + window.clearTimeout(this.hoverTimer); + if (this.playing?.hover && this.playing.video === video) this.stop(); + }, { signal }); + } + + /** After a redraw the player stays with its step while the step's card shows the same file. */ + redrawn(cards: ReadonlyMap) { + if (!this.playing) return; + const video = cards.get(this.playing.node)?.querySelector(`.fx-canvas-video[data-digest="${this.playing.preview.digest}"]`); + const frame = video?.querySelector(".fx-canvas-image-border"); + if (!video || !frame) { this.stop(); return; } + video.classList.toggle("is-shown", this.player.classList.contains("is-shown")); + Object.assign(this.playing, { video, frame }); + } + + /** Stops the video if something now covers its card. */ + recheck() { + if (this.playing && this.covered(this.playing.video)) this.stop(); + } + + stop() { + window.clearTimeout(this.hoverTimer); + const playing = this.playing; + if (!playing) return; + const focused = this.player.contains(document.activeElement); + this.playing = null; + const at = resumeAt(this.media.currentTime, this.media.duration); + if (at === null) this.positions.delete(playing.preview.digest); + else this.positions.set(playing.preview.digest, at); + cancelAnimationFrame(this.following); + this.box = ""; + this.show(playing.video, false); + this.controls.reset(); + // Paused, not unloaded: playing it again shows this frame at once and goes on from it. It fades + // out over the card's first frame, which shows again at once. + this.media.pause(); + this.hideTimer = window.setTimeout(() => { this.player.hidden = true; }, FADE); + this.unloadTimer = window.setTimeout(() => this.unload(), KEEP_LOADED); + if (focused) playing.video.querySelector(".fx-canvas-play")?.focus({ preventScroll: true }); + } + + dispose() { + this.stop(); + window.clearTimeout(this.hideTimer); + this.unload(); + this.abort.abort(); + this.observer.disconnect(); + this.waiting.clear(); + for (const still of this.stills.values()) if (still) URL.revokeObjectURL(still); + this.stills.clear(); + } + + private play(playing: Omit, hover = false) { + if (this.covered(playing.video)) return; + this.stop(); + window.clearTimeout(this.hideTimer); + window.clearTimeout(this.unloadTimer); + this.playing = { ...playing, hover }; + this.player.setAttribute("aria-label", playing.preview.label); + this.media.muted = hover; + this.media.loop = hover; + // A video stopped short of its end continues there: at once while the player still holds it, + // else from where it starts loading. + const { digest, url } = playing.preview; + const at = this.positions.get(digest) ?? 0; + if (this.media.getAttribute("src") !== url) { + this.media.src = url; + if (at) this.media.currentTime = at; + } else if (Math.abs(this.media.currentTime - at) > 0.05) this.media.currentTime = at; + this.settle(); + this.player.hidden = false; + this.follow(); + // Hovering never takes focus. + if (!hover) this.player.focus({ preventScroll: true }); + void this.media.play().catch(() => {}); + } + + /** + * The card keeps its first frame, and the player stays clear, until the video shows its own frame + * there: nothing blanks while it loads, and its controls never show a time it leaves. + */ + private settle = () => { + if (this.playing && this.media.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && !this.media.seeking) this.show(this.playing.video, true); + }; + /** Whether the video shows its frame over the card, which then hides its first frame and play control. */ + private show(video: SVGGElement, shown: boolean) { + video.classList.toggle("is-shown", shown); + this.player.classList.toggle("is-shown", shown); + } + /** Lets go of a stopped video's file and decoder. */ + private unload() { + window.clearTimeout(this.unloadTimer); + if (this.playing || !this.media.hasAttribute("src")) return; + this.media.removeAttribute("src"); + this.media.load(); + } + + /** + * Keeps the player over its card's frame every frame, as the camera and the decks move it; full + * screen, the browser places it. + */ + private follow = () => { + if (!this.playing) return; + if (!this.controls.fullscreened) { + const frame = this.playing.frame.getBoundingClientRect(), host = this.host.getBoundingClientRect(); + const box = [frame.left - host.left, frame.top - host.top, frame.width, frame.height]; + const key = box.join(); + if (key !== this.box) { + this.box = key; + (["left", "top", "width", "height"] as const).forEach((side, at) => { this.player.style[side] = `${box[at]}px`; }); + } + } + this.following = requestAnimationFrame(this.follow); + }; + + /** Asks for a file's first frame once; at most `LOADS` read at a time. */ + private read({ digest, url }: CanvasPreview) { + if (this.stills.has(digest)) return; + this.stills.set(digest, undefined); + this.queue.push({ digest, url }); + this.next(); + } + private next() { + while (this.loading < LOADS && this.queue.length) { + const { digest, url } = this.queue.shift()!; + this.loading++; + void firstFrame(url, this.abort.signal).then((blob) => { + this.loading--; + this.next(); + if (this.abort.signal.aborted) return; + const still = blob ? URL.createObjectURL(blob) : null; + this.stills.set(digest, still); + // Whichever cards show the file now; a card drawn since the read began among them. + if (still) for (const image of this.host.querySelectorAll(`.fx-canvas-video[data-digest="${digest}"] image`)) image.setAttribute("href", still); + }); + } + } +} diff --git a/web/fx-web/tests/canvas.test.ts b/web/fx-web/tests/canvas.test.ts index 8cd86eb0..2ccd459a 100644 --- a/web/fx-web/tests/canvas.test.ts +++ b/web/fx-web/tests/canvas.test.ts @@ -16,9 +16,11 @@ function report(cells: Record, order: string[] = []): cells: Object.fromEntries(Object.entries(cells).map(([address, [column, row]]) => [address, { column, row, source: "automatic" }])) }; } -test("metadata-only nodes retain their size; image nodes reserve space before loading", () => { +test("metadata-only nodes retain their size; image nodes reserve a 16:9 frame before loading", () => { expect(canvasNodeSize(node("planned"))).toEqual({ width: 250, height: 106 }); - expect(canvasNodeSize(node("rendered", true))).toEqual({ width: 250, height: 246 }); + expect(canvasNodeSize(node("rendered", true))).toEqual({ width: 250, height: 229 }); + expect(canvasNodeGeometry(node("rendered", true)).preview_height).toBe(123); + expect(canvasNodeGeometry({ ...node("ported", true), ports: { inputs: [], outputs: [], settings: [] } }).preview_height).toBe(185); }); test("declared port rows and settings reserve space above an image preview", () => { @@ -32,7 +34,7 @@ test("declared port rows and settings reserve space above an image preview", () const last = canvasPortAnchor(placed, "input", "image_11")!; expect(last.y + 16).toBeLessThan(placed.y + geometry.settings_y); expect(geometry.settings_y + 10).toBeLessThan(geometry.preview_y); - expect(geometry.preview_y + 140).toBeLessThan(geometry.height - 22); + expect(geometry.preview_y + geometry.preview_height).toBeLessThan(geometry.height - 22); }); test("mixed image and metadata nodes remain separated with edges landing at their borders", () => { @@ -200,7 +202,7 @@ test("the resting card takes the first running member and, in a take deck, the t expect(deckBadge([take("a", "x", 1), take("b", "x", 2), take("c", "y", 1)]).text).toBe("2 items · up to 2 takes"); }); -test("absent instances never join a deck, and an image output reserves its preview before it exists", () => { +test("absent instances never join a deck, and an image or video output reserves its preview before it exists", () => { const absent = { ...node("b#1", false, "b"), state: "absent" }; const layout = layoutGraph({ nodes: [node("a#1", false, "b"), absent], edges: [] }, report({ b: [0, 0] })); expect([layout.nodes.map((item) => item.id), layout.decks]).toEqual([["a#1"], []]); @@ -209,6 +211,10 @@ test("absent instances never join a deck, and an image output reserves its previ const ports = { inputs: [], outputs: [{ name: "image", type: "image/png", kind: "artifact" as const }], settings: [] }; expect(canvasPreviewReserved({ ...node("draw"), ports })).toBe(true); expect(canvasNodeSize({ ...node("draw"), ports })).toEqual(canvasNodeSize({ ...node("draw", true), ports })); + for (const type of ["video/mp4", "video", "video[]"]) { + expect(canvasPreviewReserved({ ...node("animate"), ports: { ...ports, outputs: [{ name: "clip", type, kind: "artifact" }] } })).toBe(true); + } + expect(canvasPreviewReserved({ ...node("describe"), ports: { ...ports, outputs: [{ name: "notes", type: "videos_notes", kind: "artifact" }] } })).toBe(false); }); test("a waiting member rests in front after a running one, and the badge counts the waiting", () => { diff --git a/web/fx-web/tests/export.test.ts b/web/fx-web/tests/export.test.ts index c77a1659..3969a072 100644 --- a/web/fx-web/tests/export.test.ts +++ b/web/fx-web/tests/export.test.ts @@ -70,6 +70,15 @@ describe("portable inspection boundary", () => { expect(() => parseRunExport({ ...recording.manifest, view: local })).toThrow(); expect(exportCanvasGraph(loaded(withImage(true))).nodes[0].preview).toBeUndefined(); }); + test("a recorded video previews through the same static boundary", () => { + const value = withImage(); + Object.assign(value.artifacts[0], { media_type: "video/mp4", name: "clip.mp4", path: `artifacts/${imageDigest}.mp4` }); + Object.assign(value.view.artifacts[0], { kind: "video/mp4", name: "clip.mp4" }); + value.view.nodes[0].outputs = { clip: { file: { digest: imageDigest, kind: "video/mp4", name: "clip.mp4", size: 20 } } }; + expect(exportCanvasGraph(loaded(value)).nodes[0].preview).toEqual({ + kind: "video", digest: imageDigest, url: `https://assets.example.com/runs/example/artifacts/${imageDigest}.mp4`, label: "clip: clip.mp4", count: 1, + }); + }); test("refuses path escapes, invalid omissions and inventory disagreement", () => { for (const path of ["../image.png", "/image.png", "https://example.com/image.png", "artifacts/../image.png", "artifacts/%2e%2e/image.png", "artifacts%2fimage.png", `artifacts/${imageDigest}.png?token=x`, `artifacts/${imageDigest}.png#x`, "artifacts\\image.png", `artifacts/${imageDigest}.webp`]) { const value = withImage(); value.artifacts[0].path = path; diff --git a/web/fx-web/tests/previews.test.ts b/web/fx-web/tests/previews.test.ts index 17d81697..8157a059 100644 --- a/web/fx-web/tests/previews.test.ts +++ b/web/fx-web/tests/previews.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { canvasGraph, type Artifact, type ViewerRun } from "../src/index"; +import { resumeAt } from "../src/video"; import { graph } from "./graph.test"; const sourceDigest = "a".repeat(64); @@ -25,7 +26,7 @@ function run(): ViewerRun { }; } -describe("recorded canvas image previews", () => { +describe("recorded canvas previews", () => { test("plans have no previews, including cached or planning-time results", () => { const plan = graph(); plan.instances[0].state = "done"; @@ -94,4 +95,35 @@ describe("recorded canvas image previews", () => { }; expect(canvasGraph(recorded).nodes[0].preview).toBeUndefined(); }); + + test("a step without an image output previews its first playable video", () => { + const recorded = run(); + const video = (digest: string, kind: string, name: string) => ({ file: { digest, kind, name, size: 100 } }); + recorded.nodes[0].outputs = { raw: video(sourceDigest, "video/x-matroska", "raw.mkv"), clips: { list: [video(otherDigest, "video/webm", "a.webm"), video(outputDigest, "video/mp4", "b.mp4")] } }; + recorded.artifacts = [ + artifact(sourceDigest, { kind: "video/x-matroska", name: "raw.mkv" }), + artifact(otherDigest, { kind: "video/webm", name: "a.webm" }), + artifact(outputDigest, { kind: "video/mp4", name: "b.mp4" }), + ]; + expect(canvasGraph(recorded).nodes[0].preview).toEqual({ + kind: "video", digest: otherDigest, url: `/api/artifacts/${otherDigest}`, + label: "clips: a.webm", count: 2, + }); + recorded.artifacts[1].url = "https://example.invalid/a.webm"; + expect(canvasGraph(recorded).nodes[0].preview).toMatchObject({ digest: outputDigest, count: 1 }); + }); + + test("an image output previews its card before a video output does", () => { + const recorded = run(); + recorded.nodes[0].outputs = { video: { file: { digest: otherDigest, kind: "video/mp4", name: "clip.mp4", size: 100 } }, image: file(outputDigest) }; + recorded.artifacts.push(artifact(otherDigest, { kind: "video/mp4", name: "clip.mp4" })); + expect(canvasGraph(recorded).nodes[0].preview).toMatchObject({ kind: "image", digest: outputDigest, count: 1 }); + }); + + test("a video continues where it stopped, and starts over once it has played through", () => { + expect(resumeAt(3.2, 8)).toBe(3.2); + expect([resumeAt(0, 8), resumeAt(7.9, 8), resumeAt(8, 8), resumeAt(NaN, 8)]).toEqual([null, null, null, null]); + // A stream of unknown length always continues. + expect(resumeAt(40, Infinity)).toBe(40); + }); }); diff --git a/web/fx-web/tests/video-controls.test.ts b/web/fx-web/tests/video-controls.test.ts new file mode 100644 index 00000000..c197ef05 --- /dev/null +++ b/web/fx-web/tests/video-controls.test.ts @@ -0,0 +1,22 @@ +import { expect, test } from "bun:test"; +import { playerKey, seekTarget, videoTime } from "../src/video-controls"; + +test("video times read as players show them, and an unknown length as zero", () => { + expect([0, 7.9, 64, 750, 3723, -3, NaN, Infinity].map(videoTime)) + .toEqual(["0:00", "0:07", "1:04", "12:30", "1:02:03", "0:00", "0:00", "0:00"]); +}); + +test("seeking stays within the video", () => { + expect(seekTarget(3, -5, 8)).toBe(0); + expect(seekTarget(6, 5, 8)).toBe(8); + expect(seekTarget(2, 5, 8)).toBe(7); + // A stream of unknown length seeks forward freely. + expect(seekTarget(2, 5, Infinity)).toBe(7); +}); + +test("player keys leave the player's own buttons and slider their keys", () => { + expect(["Escape", " ", "k", "K", "m", "f", "ArrowLeft", "ArrowRight", "x"].map((key) => playerKey(key, "other"))) + .toEqual(["close", "toggle", "toggle", "toggle", "mute", "fullscreen", "back", "forward", null]); + expect([" ", "Enter", "Escape"].map((key) => playerKey(key, "button"))).toEqual([null, null, "close"]); + expect(["ArrowLeft", "ArrowRight", " "].map((key) => playerKey(key, "slider"))).toEqual([null, null, "toggle"]); +}); diff --git a/web/viewer/src/main.tsx b/web/viewer/src/main.tsx index 50392260..77f97572 100644 --- a/web/viewer/src/main.tsx +++ b/web/viewer/src/main.tsx @@ -1,7 +1,7 @@ import { StrictMode, useEffect, useRef, useState, useSyncExternalStore, type ReactNode } from "react"; import { createRoot } from "react-dom/client"; import { ArrowLeft, ArrowUpRight, Check, ChevronRight, Copy, FileText, List, RefreshCw, Square, Workflow, X } from "lucide-react"; -import { ServiceIndexProjection, ViewerController, elapsedLabel, ViewerEntryController, type Artifact, type CanvasStep, type GraphDocument, type GraphInstance, type PendingRepeat, type ServiceEntry, type ServiceIndex, type ViewerRun } from "@grida/fx-web"; +import { ServiceIndexProjection, ViewerController, elapsedLabel, ownsKeys, ViewerEntryController, type Artifact, type CanvasStep, type GraphDocument, type GraphInstance, type PendingRepeat, type ServiceEntry, type ServiceIndex, type ViewerRun } from "@grida/fx-web"; import { StateIcon, StateLabel, Status, Values, WorkflowCanvas, money, type WorkflowCanvasHandle } from "@grida/fx-react"; import "./style.css"; @@ -179,7 +179,7 @@ function WorkflowView() { useEffect(() => { if (!open && !steps) return; const close = (event: KeyboardEvent) => { - if (event.key !== "Escape" || event.defaultPrevented || (event.target as Element | null)?.closest?.("input, textarea, select, [contenteditable]")) return; + if (event.key !== "Escape" || event.defaultPrevented || ownsKeys(event.target)) return; event.preventDefault(); if (open) { setAbout(false); controller.select(null); } else setSteps(false); }; diff --git a/www/README.md b/www/README.md index bfcfadde..92f94090 100644 --- a/www/README.md +++ b/www/README.md @@ -101,7 +101,8 @@ external images, including production optimizer URL validation. For a candidate: 2. Inspect browser Network and image `src`/`srcset`: external media and JSON must request the configured CDN directly, without `/_next/image?url=...` or a same-origin proxy. -3. Confirm no CORS errors and exercise video playback/seeking after host changes. +3. Confirm no CORS errors, that canvas video cards show their first frame (read from + the file, so videos need CORS too), and exercise video playback/seeking after host changes. These external asset bytes bypass Vercel optimization and delivery. HTML, JavaScript, local branding/landing media and generated social previews remain diff --git a/www/src/components/showcases/recorded-canvas-controller.ts b/www/src/components/showcases/recorded-canvas-controller.ts index a3b8e6f4..4eaa56f9 100644 --- a/www/src/components/showcases/recorded-canvas-controller.ts +++ b/www/src/components/showcases/recorded-canvas-controller.ts @@ -1,4 +1,4 @@ -import { CanvasController, ExportViewerController, exportArtifactUrl, type CanvasGraph, type ExportViewerState, type LoadedRunExport, type PinnedRunExport } from "@grida/fx-web"; +import { CanvasController, ExportViewerController, exportArtifactUrl, ownsKeys, type CanvasGraph, type ExportViewerState, type LoadedRunExport, type PinnedRunExport } from "@grida/fx-web"; export interface Recording extends PinnedRunExport { id: string; label: string } export interface CanvasFragment { graph: boolean; recording: string | null; scope: string | null; node: string | null } @@ -118,7 +118,8 @@ export class RecordedCanvasController { } }; private keydown = (event: KeyboardEvent) => { - if (event.key === "Escape" && this.viewer.getSnapshot().selected !== null && !event.defaultPrevented) { + // A text field or a card's video player keeps its own Esc. + if (event.key === "Escape" && this.viewer.getSnapshot().selected !== null && !event.defaultPrevented && !ownsKeys(event.target)) { event.preventDefault(); this.viewer.select(null); this.writeFragment(); } if (!(event.target instanceof HTMLElement) || !event.target.matches("[data-mode]")) return;