From 3cc51c4f4357a7a3157784c5f66d69dd96d648ab Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 22:25:10 +0900 Subject: [PATCH 01/10] Preview video outputs on canvas cards A step whose outputs were videos showed only its ports and status on the canvas; the video played only in the details panel. Its card now shows the video's first frame with a play button, and plays it in place. SVG has no video that every browser draws in place: Safari draws an HTML video inside foreignObject outside the SVG's transforms and above the rest of it. So the card's first frame is an ordinary SVG image, read in the browser from the file once the card comes into view (once per file, two at a time), and one HTML player laid over the card's frame plays it on request. - One rule picks a card's preview for local runs and published exports: the first image output, or without one the first MP4 or WebM output. A declared video output reserves the preview area, as an image does. - The play button is a labelled control: it neither selects the card nor opens its deck. The player follows its card as the camera and decks move, survives live redraws while the card shows the same file, and stops when it ends, on Esc, or when another card or a deck's veil covers it. - spec/layout.md, the viewing guide and the website's asset checklist (video first frames need CORS too) say so. Closes #77 --- docs/guide/07-viewing.md | 14 +- spec/layout.md | 15 ++- web/fx-web/src/canvas.css | 9 ++ web/fx-web/src/canvas.ts | 28 +++- web/fx-web/src/card.ts | 30 ++++- web/fx-web/src/export.ts | 17 +-- web/fx-web/src/graph.ts | 50 +++++--- web/fx-web/src/layout.ts | 6 +- web/fx-web/src/video.ts | 204 ++++++++++++++++++++++++++++++ web/fx-web/tests/canvas.test.ts | 6 +- web/fx-web/tests/export.test.ts | 9 ++ web/fx-web/tests/previews.test.ts | 32 ++++- www/README.md | 3 +- 13 files changed, 367 insertions(+), 56 deletions(-) create mode 100644 web/fx-web/src/video.ts diff --git a/docs/guide/07-viewing.md b/docs/guide/07-viewing.md index e81a7787..548bae42 100644 --- a/docs/guide/07-viewing.md +++ b/docs/guide/07-viewing.md @@ -269,11 +269,15 @@ automatically, polling about once a second. Reloading the page also reattaches t latest consistent snapshot. Images, text/JSON, audio, and video use generic previews; other file types can be opened in a new tab. -Run canvas cards also show the first available recorded image output. A count -indicates multiple distinct available images; select the card to inspect all outputs. -The canvas uses the verified artifact inventory and never evaluates author code to -produce thumbnails. Plan cards have no artifact previews, including steps marked -`done` by the planner. Updates preserve the canvas camera, selection and workflow navigation. +Run canvas cards also show the first available recorded image output. A step without +one shows the first frame of its first MP4 or WebM video and a play button: the video +plays over the card, follows it as you pan and zoom, and stops when it ends, on Escape, +or when another card covers it. A count indicates multiple distinct available images +(or videos); select the card to inspect all outputs. The canvas uses the verified +artifact inventory and never evaluates author code to produce thumbnails; a video's +first frame is read in the browser once its card comes into view. Plan cards have no +artifact previews, including steps marked `done` by the planner. Updates preserve the +canvas camera, selection and workflow navigation. These automatic thumbnails use structured run data. The existing authoring `view` declarations are booleans or HTML template paths; custom templates and structured diff --git a/spec/layout.md b/spec/layout.md index a904d5f7..912df8a3 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -112,8 +112,10 @@ authored cells, rule 10 removes empty tracks and rule 11 sizes them. left. Rules 1–10 read no size, so a preview arriving never changes a cell. Footprints come from -declarations: ports, settings and kind. A step whose declared output ports include an image -reserves its preview area in both views, before any artifact exists. +declarations: ports, settings and kind. A step whose declared output ports include an image or a +video reserves its preview area in both views, before any artifact exists. A recorded card +previews its first image output, or without one its first MP4 or WebM output, which shows its +first frame until it is played (§5.7). Example: `seed → propose (2 takes) → warm, cool, mono, dusk, night → sheet`, and an island `notes → index`. @@ -258,10 +260,10 @@ including those that leave the deck. | Hover a bookmark row | raise that member, as hovering its card does | the camera glides to that member's card, which is outlined; leaving keeps the camera there | | Click a bookmark row | select that member (a frame member: its first step), which then rests in front | select it, and the camera glides to its card | | Still click on a card, a back card's visible rim or the badge | expand | select that member | -| Click a labelled control on the front card (`Open workflow`) | does what it says | does what it says | +| Click a labelled control on the front card (`Open workflow`, a video's play control) | does what it says | does what it says | | Double-click | counts as one expand click | on a workflow card: open it | | Click the veil | — | collapse one level; the selection is unchanged; a second deck needs a second click | -| Esc | with focus in the canvas: go back a scope (unchanged) | collapse one level, wherever focus is except in a text field; consumed only then | +| Esc | with focus in the canvas: go back a scope (unchanged) | collapse one level, wherever focus is except in a text field or a playing video; consumed only then | | × | — | collapse one level | | Tab | one stop per deck (the front card); back cards have `tabindex=-1` | each card, in grid order | | Enter or Space | expand | select | @@ -271,6 +273,11 @@ including those that leave the deck. | Scope change | — | collapse at once; keep the pre-expand camera | | A deck inside an expanded card | — | a click expands it one level deeper. Only the innermost level is interactive; the levels around it are its veil, so a click on them collapses one level and selects nothing | +A video plays over its card's preview, following the card as the camera and the deck move it, and +selects nothing. It stops when it ends, on Esc (focus returns to its play control) and as soon as +anything covers its card: another card raised over it, its deck resting with another card in front, +or the veil of a deck it is not in. + On collapse, focus returns to the deck's front card. Raising a card never moves the focused element in the document. At most one deck per view is expanded, plus its nested levels. There are no dedicated deck buttons, no subview and no URL that selects a member (§11). diff --git a/web/fx-web/src/canvas.css b/web/fx-web/src/canvas.css index dc482bd8..07b6bcbc 100644 --- a/web/fx-web/src/canvas.css +++ b/web/fx-web/src/canvas.css @@ -49,6 +49,15 @@ @media (prefers-reduced-motion: reduce) { .fx-canvas-shimmer-band { animation: none; } } .fx-canvas-image-error { fill: #71717a; font: 11px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; pointer-events: none; } .fx-canvas-image-count { fill: #a1a1aa; font: 10px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; } +/* A video card (video.ts): its first frame, a play control, and while it plays, one player laid over the frame. */ +.fx-canvas-play { cursor: pointer; outline: none; } +.fx-canvas-play circle { fill: #18181bb3; } +.fx-canvas-play:hover circle, .fx-canvas-play:focus-visible circle { fill: #18181b; } +.fx-canvas-play:focus-visible circle { stroke: #2563eb; stroke-width: 2px; } +.fx-canvas-play svg { color: #fff; pointer-events: none; } +.fx-canvas-video.is-playing .fx-canvas-play, .fx-canvas-video.is-unavailable .fx-canvas-play { display: none; } +.fx-canvas-video.is-unavailable .fx-canvas-image-error { visibility: visible; } +.fx-canvas-player { position: absolute; box-sizing: border-box; border-radius: 6px; background: #18181b; object-fit: contain; } .fx-canvas-toolbar { position: absolute; bottom: 16px; left: 16px; display: flex; align-items: center; gap: 1px; padding: 3px; border: 1px solid #e4e4e7; border-radius: 9px; background: #fff; box-shadow: 0 2px 5px #00000008; } .fx-canvas-toolbar button { min-width: 32px; height: 30px; padding: 0 8px; border: 0; border-radius: 5px; background: #fff; color: #3f3f46; font: 13px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; cursor: pointer; } .fx-canvas-toolbar button:hover { background: #f4f4f5; } diff --git a/web/fx-web/src/canvas.ts b/web/fx-web/src/canvas.ts index 4e33acad..b1646e74 100644 --- a/web/fx-web/src/canvas.ts +++ b/web/fx-web/src/canvas.ts @@ -5,8 +5,11 @@ import { displayPortName } from "./ports"; import { BADGE_BAND, DECK_OFFSET, canvasEdgePath, layoutGraph, type CanvasLayout, type PlacedDeck, type PositionedFrame, type PositionedNode } from "./layout"; import { DECK_BACK, bookmarkLayout, deckBadge, deckStack, deckTicks, expandedColumns, restingFront, tickHolds, type DeckMember } from "./deck"; import { element, nodeCard, runningLabel, short } from "./card"; +import { CardVideos } from "./video"; const EXPANDED_GAP = 32; +/** A card's labelled controls do what they say (§5.7): a press on one neither pans nor opens its deck. */ +const CARD_CONTROLS = ".fx-canvas-open-workflow, .fx-canvas-play"; /** §5.9: expanded decks fit the viewport down to card titles (13 px) about 11 px on screen. */ const LEGIBLE_ZOOM = 11 / 13; @@ -84,6 +87,7 @@ export class CanvasController { private readonly deckBar = document.createElement("div"); /** An open deck's bookmark, fixed at the canvas's left edge while its grid moves. */ private readonly scrubber = document.createElement("div"); + private readonly videos: CardVideos; /** The badge's text size on screen last written to `--fx-text`, within its fade. */ private textSize = ""; private layout: CanvasLayout = { nodes: [], edges: [], frames: [], decks: [], width: 0, height: 0 }; @@ -158,20 +162,23 @@ export class CanvasController { const index = this.open ? this.decks.indexOf(this.open.deck) : -1; if (index >= 0) this.markFront(this.decks[index], this.frontOf(index)); }, { signal: this.abort.signal }); - container.replaceChildren(this.svg, this.toolbar, this.empty, this.deckBar, this.scrubber); + this.videos = new CardVideos(container); + container.replaceChildren(this.svg, this.videos.player, this.toolbar, this.empty, this.deckBar, this.scrubber); container.addEventListener("wheel", this.onWheel, { passive: false, signal: this.abort.signal }); this.svg.addEventListener("pointerdown", this.onPointerDown, { signal: this.abort.signal }); this.svg.addEventListener("pointermove", this.onPointerMove, { signal: this.abort.signal }); this.svg.addEventListener("pointerup", this.onPointerUp, { signal: this.abort.signal }); this.svg.addEventListener("pointercancel", this.onPointerUp, { signal: this.abort.signal }); - this.svg.addEventListener("pointerleave", () => this.hover(null), { signal: this.abort.signal }); + // The pointer moving onto the player stays on its card. + this.svg.addEventListener("pointerleave", (event) => { if (event.relatedTarget !== this.videos.player) this.hover(null); }, { signal: this.abort.signal }); // A deck card is the expand target: the capture phase runs before a card's own selection. this.svg.addEventListener("click", this.onDeckClick, { capture: true, signal: this.abort.signal }); // Raising a card moves focus out of the canvas, so Esc listens on the document while open. document.addEventListener("keydown", (event) => { if (event.key !== "Escape" || !this.open || event.defaultPrevented) return; const target = event.target as HTMLElement | null; - if (target?.closest("input, textarea, select, [contenteditable]")) return; + // A field keeps its own keys; Esc on the card's player closes the player first. + if (target?.closest("input, textarea, select, video, [contenteditable]")) return; event.preventDefault(); this.collapse(); }, { capture: true, signal: this.abort.signal }); @@ -264,6 +271,7 @@ export class CanvasController { } for (const [index, node] of this.layout.nodes.entries()) { const group = nodeCard(node, this.id, index, signal, (scope) => this.onOpenScope(scope)); + if (node.preview?.kind === "video") this.videos.attach(group, node.id, node.preview, signal); group.addEventListener("pointerenter", () => this.highlightConnections(node.id), { signal }); group.addEventListener("pointerleave", () => this.highlightConnections(null), { signal }); group.addEventListener("click", () => this.onSelect(node.id), { signal }); @@ -277,6 +285,7 @@ export class CanvasController { layer(frameOf.get(node.id) ?? null).cards.append(group); this.nodes.set(node.id, group); } + this.videos.redrawn(this.nodes); const items = new Map([...this.layout.nodes, ...frames].map((item) => [item.id, item])); this.decks = this.layout.decks.map((placed, index) => this.deck(placed, index, layer(placed.frame).cards, items)); this.decks.forEach((_, index) => this.restDeck(index)); @@ -469,6 +478,9 @@ export class CanvasController { for (const at of stack.order) if (stack.shown[at]) deck.parent.insertBefore(deck.cards[at], deck.badge); this.stack(deck, stack.slots); deck.cards.forEach((card, at) => card.classList.toggle("is-buried", !stack.shown[at])); + // Only the front card is uncovered. + const playing = this.videos.card; + if (playing && deck.cards.some((card, at) => at !== front && card.contains(playing))) this.videos.stop(); this.markFront(deck, front); // One tab stop per closed deck: its front card. deck.cards.forEach((card, at) => { if (card.hasAttribute("tabindex")) card.setAttribute("tabindex", at === front ? "0" : "-1"); }); @@ -544,8 +556,7 @@ export class CanvasController { if (target) this.restDeck(target.deck, target.card, !target.mark); } private onDeckClick = (event: MouseEvent) => { - // A labelled control on a card does what it says (§5.7). - if ((event.target as Element).closest(".fx-canvas-open-workflow")) return; + if ((event.target as Element).closest(CARD_CONTROLS)) return; const badge = (event.target as Element).closest(".fx-deck-badge"); const hit = badge ? { deck: Number(badge.dataset.deck), card: 0 } : this.cardAt(event.target); if (!hit || this.open) return; @@ -561,6 +572,9 @@ export class CanvasController { const deck = this.decks[index]; if (!deck || this.open) return; this.hover(null); + // The veil covers every card outside the deck. + const playing = this.videos.card; + if (playing && !deck.cards.some((card) => card.contains(playing))) this.videos.stop(); const sizes = deck.boxes; const card = { width: Math.max(...sizes.map((size) => size.width)), height: Math.max(...sizes.map((size) => size.height)) }; const bounds = this.container.getBoundingClientRect(); @@ -740,7 +754,7 @@ export class CanvasController { */ private onPointerDown = (event: PointerEvent) => { const target = event.target as Element; - if (event.button !== 0 || target.closest(".fx-canvas-open-workflow")) return; + if (event.button !== 0 || target.closest(CARD_CONTROLS)) return; this.drag = { pointer: event.pointerId, x: event.clientX, y: event.clientY, startX: event.clientX, startY: event.clientY, moved: false, background: !target.closest("[data-node-id], [data-connection-id], [data-deck], .fx-deck-bookmark"), veil: Boolean(target.closest(".fx-deck-veil")) }; }; @@ -780,6 +794,6 @@ export class CanvasController { dispose() { if (this.viewportFrame !== null) cancelAnimationFrame(this.viewportFrame); this.viewportFrame = null; - this.cancelDrag(); this.abort.abort(); this.graphAbort.abort(); this.resize.disconnect(); this.container.replaceChildren(); + this.cancelDrag(); this.abort.abort(); this.graphAbort.abort(); this.videos.dispose(); this.resize.disconnect(); this.container.replaceChildren(); } } diff --git a/web/fx-web/src/card.ts b/web/fx-web/src/card.ts index 3ce3ac50..3b3f6efe 100644 --- a/web/fx-web/src/card.ts +++ b/web/fx-web/src/card.ts @@ -1,3 +1,4 @@ +import { createElement, Play } from "lucide"; import { displayPortName } from "./ports"; import { canvasNodeGeometry, canvasPreviewReserved, PORT_ROW, PORT_TOP, type PositionedNode } from "./layout"; import { stateIcon, stateIconParts } from "./state-icon"; @@ -61,7 +62,8 @@ export function nodeCard(node: PositionedNode, canvasId: string, index: number, const portsLabel = node.ports ? `, inputs: ${node.ports.inputs.map((port) => `${displayPortName(port.name)} (${port.type}, ${port.kind})`).join(", ") || "none"}, outputs: ${node.ports.outputs.map((port) => `${displayPortName(port.name)} (${port.type}, ${port.kind})`).join(", ") || "none"}` : node.pending ? "" : ", port metadata not recorded"; const scopeLabel = node.kind === "workflow" ? `, imported workflow, ${node.child_count ?? 0} steps, ${node.failure_count ?? 0} failed` : ""; const state = node.state === "waiting" && node.waiting_for ? `waiting for ${node.waiting_for}` : node.state; - const label = `${node.title}, ${state}${scopeLabel}${portsLabel}${preview ? `, ${preview.label}, ${preview.count} image ${preview.count === 1 ? "output" : "outputs"}` : ""}`; + const count = preview ? `${preview.count} ${preview.kind} ${preview.count === 1 ? "output" : "outputs"}` : ""; + const label = `${node.title}, ${state}${scopeLabel}${portsLabel}${preview ? `, ${preview.label}, ${count}` : ""}`; const group = element("g", { transform: `translate(${node.x},${node.y})`, class: `fx-canvas-node${node.kind === "workflow" ? " fx-canvas-workflow" : node.kind === "boundary" ? " fx-canvas-boundary" : ""}`, role: node.kind === "workflow" ? "group" : "button", tabindex: 0, "data-node-id": node.id, "data-state": node.state, "aria-label": label, ...(node.kind === "workflow" ? { "aria-current": "false" } : { "aria-pressed": "false" }) }); group.append(element("rect", { width: node.width, height: node.height, rx: 11, class: node.pending ? "fx-canvas-node-box fx-canvas-pending" : "fx-canvas-node-box" })); group.append(element("text", { x: 16, y: 27, class: "fx-canvas-node-title" }, short(node.title, node.ports ? 42 : 28))); @@ -99,19 +101,33 @@ export function nodeCard(node: PositionedNode, canvasId: string, index: number, const clip = element("clipPath", { id: clipId }); clip.append(element("rect", { ...frame, rx: 6 })); group.append(clip, element("rect", { ...frame, rx: 6, fill: `url(#${canvasId}-checker)` })); + const unavailable = preview.kind === "video" ? "Video unavailable" : "Image unavailable"; const image = element("image", { ...frame, preserveAspectRatio: "xMidYMid meet", "clip-path": `url(#${clipId})`, role: "img", "aria-label": preview.label, class: "fx-canvas-node-image" }); - const fallback = element("text", { x: node.width / 2, y: frame.y + frame.height / 2, "text-anchor": "middle", "dominant-baseline": "middle", class: "fx-canvas-image-error", visibility: "hidden" }, "Image unavailable"); + const fallback = element("text", { x: node.width / 2, y: frame.y + frame.height / 2, "text-anchor": "middle", "dominant-baseline": "middle", class: "fx-canvas-image-error", visibility: "hidden" }, unavailable); image.addEventListener("error", () => { image.setAttribute("visibility", "hidden"); fallback.setAttribute("visibility", "visible"); - group.setAttribute("aria-label", `${node.title}, ${state}${portsLabel}, Image unavailable: ${preview.label}`); + group.setAttribute("aria-label", `${node.title}, ${state}${portsLabel}, ${unavailable}: ${preview.label}`); }, { signal }); - image.setAttribute("href", preview.url); - group.append(image, element("rect", { ...frame, rx: 6, class: "fx-canvas-image-border" }), fallback); - group.append(element("text", { x: node.width - 16, y: node.height - 22, "text-anchor": "end", class: "fx-canvas-image-count" }, `${preview.count} image ${preview.count === 1 ? "output" : "outputs"}`)); + const border = element("rect", { ...frame, rx: 6, class: "fx-canvas-image-border" }); + if (preview.kind === "image") { + image.setAttribute("href", preview.url); + group.append(image, border, fallback); + } else { + // A video shows its first frame, which the canvas sets once the card is in view, and a play + // control; the canvas plays it over the frame (video.ts). + const video = element("g", { class: "fx-canvas-video", "data-digest": preview.digest }); + const play = element("g", { class: "fx-canvas-play", role: "button", tabindex: 0, "aria-label": `Play ${preview.label}` }); + const center = { x: node.width / 2, y: frame.y + frame.height / 2 }; + play.append(element("circle", { cx: center.x, cy: center.y, r: 18 }), + createElement(Play, { x: center.x - 8, y: center.y - 8, width: 16, height: 16, "stroke-width": 2, "aria-hidden": "true" })); + video.append(image, border, fallback, play); + group.append(video); + } + group.append(element("text", { x: node.width - 16, y: node.height - 22, "text-anchor": "end", class: "fx-canvas-image-count" }, count)); } else if (canvasPreviewReserved(node)) { const frame = { x: 16, y: canvasNodeGeometry(node).preview_y, width: node.width - 32, height: 140 }; - // Only a started step shimmers: the declared image may be on its way. Waiting and settled steps keep the empty frame. + // Only a started step shimmers: the declared file may be on its way. Waiting and settled steps keep the empty frame. if (node.state === "running") { const clipId = `${canvasId}-shimmer-${index}`; const clip = element("clipPath", { id: clipId }); diff --git a/web/fx-web/src/export.ts b/web/fx-web/src/export.ts index c6345278..96b486c5 100644 --- a/web/fx-web/src/export.ts +++ b/web/fx-web/src/export.ts @@ -1,5 +1,5 @@ import { parseViewerRun, type ViewerRun } from "./index"; -import { flatCanvasGraph, parseLayoutReport, type CanvasGraph, type LayoutReport } from "./graph"; +import { cardPreview, flatCanvasGraph, parseLayoutReport, type CanvasGraph, type LayoutReport } from "./graph"; import { projectScopes } from "./scopes"; export interface ExportArtifact { @@ -28,7 +28,6 @@ export interface LoadedRunExport { manifest: RunExport; manifest_url: string } const record = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); const digest = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); const count = (value: unknown): value is number => typeof value === "number" && Number.isSafeInteger(value) && value >= 0; -const imageTypes = new Set(["image/png", "image/jpeg", "image/webp", "image/gif", "image/avif"]); const mediaExtensions: Record = { "image/png": "png", "image/jpeg": "jpg", "image/webp": "webp", "image/gif": "gif", "image/avif": "avif", "video/mp4": "mp4", "video/webm": "webm", "audio/wav": "wav", "audio/mpeg": "mp3", "audio/ogg": "ogg", "audio/flac": "flac", "model/gltf-binary": "glb", @@ -223,7 +222,7 @@ export async function readRunExport(pin: PinnedRunExport, signal?: AbortSignal, return { manifest: parseRunExport(value), manifest_url: url.href }; } -/** Share wiring, scope projection and geometry while resolving images through the static boundary. */ +/** Share wiring, scope projection and geometry while resolving previews through the static boundary. */ export function exportCanvasGraph(loaded: LoadedRunExport, scope: string | null = null): CanvasGraph { const { view, artifacts } = loaded.manifest; const flat = flatCanvasGraph(view); @@ -231,14 +230,12 @@ export function exportCanvasGraph(loaded: LoadedRunExport, scope: string | null for (const node of flat.nodes) { const source = view.nodes.find((item) => item.id === node.source_id); if (!source) continue; - const images = new Map(); - for (const [port, value] of Object.entries(source.outputs)) for (const id of exportFileDigests(value)) { + const preview = cardPreview(Object.entries(source.outputs).flatMap(([port, value]) => exportFileDigests(value).flatMap((id) => { const artifact = inventory.get(id); - const url = exportArtifactUrl(loaded, id); - if (artifact && imageTypes.has(artifact.media_type) && url && !images.has(id)) images.set(id, { url, label: `${port}: ${artifact.name}` }); - } - const first = images.entries().next().value; - if (first) node.preview = { kind: "image", digest: first[0], ...first[1], count: images.size }; + const url = artifact ? exportArtifactUrl(loaded, id) : null; + return artifact && url ? [{ digest: id, media_type: artifact.media_type, url, label: `${port}: ${artifact.name}` }] : []; + }))); + if (preview) node.preview = preview; } return projectScopes(view, flat, scope); } diff --git a/web/fx-web/src/graph.ts b/web/fx-web/src/graph.ts index 3115be93..997e2109 100644 --- a/web/fx-web/src/graph.ts +++ b/web/fx-web/src/graph.ts @@ -150,8 +150,11 @@ export function parseLayoutReport(value: unknown): LayoutReport { return value as unknown as LayoutReport; } -export interface CanvasImagePreview { - kind: "image"; digest: string; url: string; label: string; count: number; +/** A card's preview: an image draws as itself, a video as its first frame until it is played. */ +export interface CanvasPreview { + kind: "image" | "video"; digest: string; url: string; label: string; + /** The step's distinct output files of the preview's kind. */ + count: number; } export interface CanvasStep { id: string; source_id: string; title: string; subtitle: string; state: string; pending: boolean; @@ -175,7 +178,7 @@ export interface CanvasStep { child_count?: number; failure_count?: number; errors?: { id: string; title: string; message: string }[]; - preview?: CanvasImagePreview; + preview?: CanvasPreview; ports?: { inputs: CanvasPort[]; outputs: CanvasPort[]; settings: { name: string; type: string }[] }; } export interface CanvasPort { name: string; type: string; kind: "artifact" | "parameter" | "fact" | "value" } @@ -187,7 +190,11 @@ export interface CanvasConnection { export interface CanvasFrame { id: string; title: string; nodes: string[]; parent?: string | null; address?: string; take?: number[]; key?: string; state?: string } export interface CanvasGraph { nodes: CanvasStep[]; edges: CanvasConnection[]; frames?: CanvasFrame[] } -const imageKinds = new Set(["image/png", "image/jpeg", "image/webp", "image/gif", "image/avif"]); +/** The files a card previews: pictures and the videos every browser plays, never active documents. */ +const previewKinds: Record = { + "image/png": "image", "image/jpeg": "image", "image/webp": "image", "image/gif": "image", "image/avif": "image", + "video/mp4": "video", "video/webm": "video", +}; const compareText = (left: string, right: string) => left < right ? -1 : left > right ? 1 : 0; const compareNamed = (left: { name: string; kind?: string }, right: { name: string; kind?: string }) => compareText(left.name, right.name) || compareText(left.kind ?? "", right.kind ?? ""); @@ -202,19 +209,28 @@ function recordedFileDigests(value: unknown): string[] { return []; } -/** Select a recorded output image from the host's verified artifact inventory. */ -function imagePreview(node: ViewerRun["nodes"][number], artifacts: Map, apiBase: string): CanvasImagePreview | undefined { - const images = new Map(); - for (const [port, value] of Object.entries(node.outputs)) { - for (const digest of recordedFileDigests(value)) { - const artifact = artifacts.get(digest); - if (artifact?.available && imageKinds.has(artifact.kind) && artifact.url === `${apiBase}/artifacts/${digest}` && !images.has(digest)) { - images.set(digest, { url: artifact.url, label: `${port}: ${artifact.name}` }); - } - } +/** + * A card's preview from its output files in output order: the first image, or without one the + * first video. Callers pass only files they resolved to a verified URL. + */ +export function cardPreview(files: Iterable<{ digest: string; media_type: string; url: string; label: string }>): CanvasPreview | undefined { + const found = { image: new Map(), video: new Map() }; + for (const { digest, media_type, url, label } of files) { + const kind = previewKinds[media_type]; + if (kind && !found[kind].has(digest)) found[kind].set(digest, { url, label }); } - const first = images.entries().next().value; - return first ? { kind: "image", digest: first[0], ...first[1], count: images.size } : undefined; + const kind = found.image.size ? "image" : found.video.size ? "video" : undefined; + const first = kind && found[kind].entries().next().value; + return kind && first ? { kind, digest: first[0], ...first[1], count: found[kind].size } : undefined; +} + +/** Select a recorded output picture or video from the host's verified artifact inventory. */ +function recordedPreview(node: ViewerRun["nodes"][number], artifacts: Map, apiBase: string): CanvasPreview | undefined { + return cardPreview(Object.entries(node.outputs).flatMap(([port, value]) => recordedFileDigests(value).flatMap((digest) => { + const artifact = artifacts.get(digest); + return artifact?.available && artifact.url === `${apiBase}/artifacts/${digest}` + ? [{ digest, media_type: artifact.kind, url: artifact.url, label: `${port}: ${artifact.name}` }] : []; + }))); } /** Project declared wiring for display; no expressions or execution decisions are evaluated. */ @@ -224,7 +240,7 @@ export function flatCanvasGraph(view: GraphDocument | ViewerRun, apiBase = "/api const artifacts = new Map(plan ? [] : view.artifacts.map((item) => [item.digest, item])); const source = plan ? view.instances : view.nodes; const nodes: CanvasStep[] = source.map((item) => { - const preview = plan ? undefined : imagePreview(item as ViewerRun["nodes"][number], artifacts, apiBase); + const preview = plan ? undefined : recordedPreview(item as ViewerRun["nodes"][number], artifacts, apiBase); const run = plan ? undefined : item as ViewerRun["nodes"][number]; const started_at = run?.started_at; const declared = plan ? view.types?.[item.uses ?? ""]?.ports : (item as ViewerRun["nodes"][number]).ports; diff --git a/web/fx-web/src/layout.ts b/web/fx-web/src/layout.ts index 05c5d517..7a294e2f 100644 --- a/web/fx-web/src/layout.ts +++ b/web/fx-web/src/layout.ts @@ -13,12 +13,12 @@ const COLUMN_GAP = 100, ROW_GAP = 42, FRAME_PAD = 20, FRAME_TOP = 36, MARGIN = 2 /** Socket rows: the card draws them and edges attach to them. */ export const PORT_TOP = 93, PORT_ROW = 34; -/** §4: a card with a declared image output reserves its preview area before any image exists. */ +/** §4: a card with a declared image or video output reserves its preview area before any file exists. */ export function canvasPreviewReserved(node: CanvasStep) { - return node.preview !== undefined || (node.ports?.outputs.some((port) => port.kind === "artifact" && /^image(?:[/[{?]|$)/.test(port.type)) ?? false); + return node.preview !== undefined || (node.ports?.outputs.some((port) => port.kind === "artifact" && /^(?:image|video)(?:[/[{?]|$)/.test(port.type)) ?? false); } -/** Geometry depends on declarations, never on image loading or measured DOM text. */ +/** Geometry depends on declarations, never on media loading or measured DOM text. */ export function canvasNodeGeometry(node: CanvasStep) { if (!node.ports) return { width: 250, height: node.preview ? 246 : 106, preview_y: 61, settings_y: 0 }; const rows = Math.max(node.ports.inputs.length, node.ports.outputs.length); diff --git a/web/fx-web/src/video.ts b/web/fx-web/src/video.ts new file mode 100644 index 00000000..f076fcf4 --- /dev/null +++ b/web/fx-web/src/video.ts @@ -0,0 +1,204 @@ +import type { CanvasPreview } from "./graph"; + +/** The widest first frame kept, in pixels: a card's frame is 328 canvas units wide. */ +const STILL_WIDTH = 640; +/** How many videos load at once for their first frames. */ +const LOADS = 2; + +type Box = Pick; + +/** Where the player goes, in the host's pixels: over the frame as it is on screen now. */ +export function playerBox(frame: Box, host: Box) { + return { left: frame.left - host.left, top: frame.top - host.top, width: frame.width, height: frame.height }; +} + +/** + * A video's first frame as a picture, or null when the video does not load or this page may not + * read its pixels: a file on another host must allow this page (CORS), or it does not load. + */ +async function firstFrame(url: string, signal: AbortSignal): Promise { + const video = document.createElement("video"); + video.crossOrigin = "anonymous"; + video.muted = true; + video.playsInline = true; + video.preload = "auto"; + try { + const loaded = await new Promise((resolve) => { + video.addEventListener("loadeddata", () => resolve(true), { once: true }); + video.addEventListener("error", () => resolve(false), { once: true }); + signal.addEventListener("abort", () => resolve(false), { once: true }); + video.src = url; + }); + if (!loaded || !video.videoWidth || !video.videoHeight) return null; + const scale = Math.min(1, STILL_WIDTH / video.videoWidth); + const canvas = document.createElement("canvas"); + canvas.width = Math.max(1, Math.round(video.videoWidth * scale)); + canvas.height = Math.max(1, Math.round(video.videoHeight * scale)); + canvas.getContext("2d")?.drawImage(video, 0, 0, canvas.width, canvas.height); + // WebP keeps transparency; a browser that cannot write it gives PNG. + return await new Promise((resolve) => canvas.toBlob(resolve, "image/webp")); + } catch { + return null; + } finally { + video.removeAttribute("src"); + video.load(); + } +} + +/** + * Video previews on node cards. SVG has no video that every browser draws in place: Safari draws + * an HTML video inside `foreignObject` outside the SVG's transforms and above the rest of it. So a + * card shows its video's first frame as an ordinary SVG image, read once per file when the card + * first comes into view, and one HTML player, laid over the card's frame, plays it on request. + * The canvas stops the player when anything covers its card. + */ +export class CardVideos { + /** First frames by digest: an object URL, or null when the file could not be read. */ + private readonly stills = new Map>(); + private readonly ready = new Map(); + /** Files the player could not play: their cards say so and offer no play control. */ + private readonly failed = new Set(); + private readonly waiting = new Map void>(); + private readonly observer = new IntersectionObserver((entries) => { + for (const entry of entries) { + const load = entry.isIntersecting ? this.waiting.get(entry.target) : undefined; + if (!load) continue; + this.observer.unobserve(entry.target); + this.waiting.delete(entry.target); + load(); + } + }, { rootMargin: "200px" }); + private readonly queue: (() => void)[] = []; + private loading = 0; + private readonly abort = new AbortController(); + readonly player = document.createElement("video"); + private playing: { node: string; preview: CanvasPreview; card: SVGGElement; video: SVGGElement } | null = null; + private frame: number | null = null; + + constructor(private readonly host: HTMLElement) { + const signal = this.abort.signal; + this.player.className = "fx-canvas-player"; + this.player.controls = true; + this.player.playsInline = true; + this.player.hidden = true; + this.player.addEventListener("ended", () => this.stop(), { signal }); + this.player.addEventListener("error", () => { + if (!this.playing || !this.player.getAttribute("src")) return; + this.failed.add(this.playing.preview.digest); + this.playing.video.classList.add("is-unavailable"); + this.stop(); + }, { signal }); + this.player.addEventListener("keydown", (event) => { + if (event.key !== "Escape" || event.defaultPrevented) return; + event.preventDefault(); + event.stopPropagation(); + this.stop(); + }, { signal }); + } + + /** The card whose video plays. */ + get card(): SVGGElement | null { return this.playing?.card ?? null; } + + /** Wires a drawn card's video: its first frame once the card is in view, and its play control. */ + attach(card: SVGGElement, node: string, preview: CanvasPreview, signal: AbortSignal) { + const video = card.querySelector(".fx-canvas-video"); + const image = video?.querySelector("image"); + const play = video?.querySelector(".fx-canvas-play"); + if (!video || !image || !play) return; + if (this.failed.has(preview.digest)) video.classList.add("is-unavailable"); + const show = (url: string | null) => { if (url && !signal.aborted) image.setAttribute("href", url); }; + if (this.ready.has(preview.digest)) show(this.ready.get(preview.digest)!); + else { + this.waiting.set(image, () => void this.still(preview).then(show)); + this.observer.observe(image); + signal.addEventListener("abort", () => { this.observer.unobserve(image); this.waiting.delete(image); }, { once: true }); + } + // A labelled control: it plays, and neither selects the card nor opens its deck. + const start = (event: Event) => { + event.preventDefault(); + event.stopPropagation(); + this.play({ node, preview, card, video }); + }; + play.addEventListener("click", start, { signal }); + play.addEventListener("keydown", (event) => { if (event.key === "Enter" || event.key === " ") start(event); }, { signal }); + } + + /** After a redraw the player stays with its step while the step's card shows the same file. */ + redrawn(cards: ReadonlyMap) { + if (!this.playing) return; + const card = cards.get(this.playing.node); + const video = card?.querySelector(`.fx-canvas-video[data-digest="${this.playing.preview.digest}"]`); + if (!card || !video) { this.stop(); return; } + video.classList.add("is-playing"); + Object.assign(this.playing, { card, video }); + } + + stop() { + const playing = this.playing; + if (!playing) return; + const focused = document.activeElement === this.player; + this.playing = null; + if (this.frame !== null) cancelAnimationFrame(this.frame); + this.frame = null; + playing.video.classList.remove("is-playing"); + this.player.pause(); + this.player.hidden = true; + this.player.removeAttribute("src"); + this.player.load(); + if (focused) playing.video.querySelector(".fx-canvas-play")?.focus({ preventScroll: true }); + } + + dispose() { + this.stop(); + this.abort.abort(); + this.observer.disconnect(); + this.waiting.clear(); + for (const url of this.ready.values()) if (url) URL.revokeObjectURL(url); + this.ready.clear(); + } + + private play(playing: NonNullable) { + this.stop(); + this.playing = playing; + playing.video.classList.add("is-playing"); + this.player.setAttribute("aria-label", playing.preview.label); + this.player.src = playing.preview.url; + this.player.hidden = false; + this.follow(); + this.player.focus({ preventScroll: true }); + void this.player.play().catch(() => {}); + } + + /** Keeps the player over its card's frame every frame, as the camera and the decks move it. */ + private follow = () => { + const frame = this.playing?.video.querySelector(".fx-canvas-image-border"); + if (!frame) return; + const box = playerBox(frame.getBoundingClientRect(), this.host.getBoundingClientRect()); + for (const side of ["left", "top", "width", "height"] as const) this.player.style[side] = `${box[side]}px`; + this.frame = requestAnimationFrame(this.follow); + }; + + /** A file's first frame, read once and at most `LOADS` at a time. */ + private still(preview: CanvasPreview): Promise { + let still = this.stills.get(preview.digest); + if (still) return still; + const signal = this.abort.signal; + still = new Promise((resolve) => { this.queue.push(resolve); this.next(); }) + .then(() => firstFrame(preview.url, signal)) + .then((blob) => { + this.loading--; + this.next(); + const url = blob && !signal.aborted ? URL.createObjectURL(blob) : null; + this.ready.set(preview.digest, url); + return url; + }); + this.stills.set(preview.digest, still); + return still; + } + private next() { + while (this.loading < LOADS && this.queue.length) { + this.loading++; + this.queue.shift()!(); + } + } +} diff --git a/web/fx-web/tests/canvas.test.ts b/web/fx-web/tests/canvas.test.ts index 8cd86eb0..2a6e28c6 100644 --- a/web/fx-web/tests/canvas.test.ts +++ b/web/fx-web/tests/canvas.test.ts @@ -200,7 +200,7 @@ test("the resting card takes the first running member and, in a take deck, the t expect(deckBadge([take("a", "x", 1), take("b", "x", 2), take("c", "y", 1)]).text).toBe("2 items · up to 2 takes"); }); -test("absent instances never join a deck, and an image output reserves its preview before it exists", () => { +test("absent instances never join a deck, and an image or video output reserves its preview before it exists", () => { const absent = { ...node("b#1", false, "b"), state: "absent" }; const layout = layoutGraph({ nodes: [node("a#1", false, "b"), absent], edges: [] }, report({ b: [0, 0] })); expect([layout.nodes.map((item) => item.id), layout.decks]).toEqual([["a#1"], []]); @@ -209,6 +209,10 @@ test("absent instances never join a deck, and an image output reserves its previ const ports = { inputs: [], outputs: [{ name: "image", type: "image/png", kind: "artifact" as const }], settings: [] }; expect(canvasPreviewReserved({ ...node("draw"), ports })).toBe(true); expect(canvasNodeSize({ ...node("draw"), ports })).toEqual(canvasNodeSize({ ...node("draw", true), ports })); + for (const type of ["video/mp4", "video", "video[]"]) { + expect(canvasPreviewReserved({ ...node("animate"), ports: { ...ports, outputs: [{ name: "clip", type, kind: "artifact" }] } })).toBe(true); + } + expect(canvasPreviewReserved({ ...node("describe"), ports: { ...ports, outputs: [{ name: "notes", type: "videos_notes", kind: "artifact" }] } })).toBe(false); }); test("a waiting member rests in front after a running one, and the badge counts the waiting", () => { diff --git a/web/fx-web/tests/export.test.ts b/web/fx-web/tests/export.test.ts index c77a1659..3969a072 100644 --- a/web/fx-web/tests/export.test.ts +++ b/web/fx-web/tests/export.test.ts @@ -70,6 +70,15 @@ describe("portable inspection boundary", () => { expect(() => parseRunExport({ ...recording.manifest, view: local })).toThrow(); expect(exportCanvasGraph(loaded(withImage(true))).nodes[0].preview).toBeUndefined(); }); + test("a recorded video previews through the same static boundary", () => { + const value = withImage(); + Object.assign(value.artifacts[0], { media_type: "video/mp4", name: "clip.mp4", path: `artifacts/${imageDigest}.mp4` }); + Object.assign(value.view.artifacts[0], { kind: "video/mp4", name: "clip.mp4" }); + value.view.nodes[0].outputs = { clip: { file: { digest: imageDigest, kind: "video/mp4", name: "clip.mp4", size: 20 } } }; + expect(exportCanvasGraph(loaded(value)).nodes[0].preview).toEqual({ + kind: "video", digest: imageDigest, url: `https://assets.example.com/runs/example/artifacts/${imageDigest}.mp4`, label: "clip: clip.mp4", count: 1, + }); + }); test("refuses path escapes, invalid omissions and inventory disagreement", () => { for (const path of ["../image.png", "/image.png", "https://example.com/image.png", "artifacts/../image.png", "artifacts/%2e%2e/image.png", "artifacts%2fimage.png", `artifacts/${imageDigest}.png?token=x`, `artifacts/${imageDigest}.png#x`, "artifacts\\image.png", `artifacts/${imageDigest}.webp`]) { const value = withImage(); value.artifacts[0].path = path; diff --git a/web/fx-web/tests/previews.test.ts b/web/fx-web/tests/previews.test.ts index 17d81697..9495632d 100644 --- a/web/fx-web/tests/previews.test.ts +++ b/web/fx-web/tests/previews.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { canvasGraph, type Artifact, type ViewerRun } from "../src/index"; +import { playerBox } from "../src/video"; import { graph } from "./graph.test"; const sourceDigest = "a".repeat(64); @@ -25,7 +26,7 @@ function run(): ViewerRun { }; } -describe("recorded canvas image previews", () => { +describe("recorded canvas previews", () => { test("plans have no previews, including cached or planning-time results", () => { const plan = graph(); plan.instances[0].state = "done"; @@ -94,4 +95,33 @@ describe("recorded canvas image previews", () => { }; expect(canvasGraph(recorded).nodes[0].preview).toBeUndefined(); }); + + test("a step without an image output previews its first playable video", () => { + const recorded = run(); + const video = (digest: string, kind: string, name: string) => ({ file: { digest, kind, name, size: 100 } }); + recorded.nodes[0].outputs = { raw: video(sourceDigest, "video/x-matroska", "raw.mkv"), clips: { list: [video(otherDigest, "video/webm", "a.webm"), video(outputDigest, "video/mp4", "b.mp4")] } }; + recorded.artifacts = [ + artifact(sourceDigest, { kind: "video/x-matroska", name: "raw.mkv" }), + artifact(otherDigest, { kind: "video/webm", name: "a.webm" }), + artifact(outputDigest, { kind: "video/mp4", name: "b.mp4" }), + ]; + expect(canvasGraph(recorded).nodes[0].preview).toEqual({ + kind: "video", digest: otherDigest, url: `/api/artifacts/${otherDigest}`, + label: "clips: a.webm", count: 2, + }); + recorded.artifacts[1].url = "https://example.invalid/a.webm"; + expect(canvasGraph(recorded).nodes[0].preview).toMatchObject({ digest: outputDigest, count: 1 }); + }); + + test("an image output previews its card before a video output does", () => { + const recorded = run(); + recorded.nodes[0].outputs = { video: { file: { digest: otherDigest, kind: "video/mp4", name: "clip.mp4", size: 100 } }, image: file(outputDigest) }; + recorded.artifacts.push(artifact(otherDigest, { kind: "video/mp4", name: "clip.mp4" })); + expect(canvasGraph(recorded).nodes[0].preview).toMatchObject({ kind: "image", digest: outputDigest, count: 1 }); + }); + + test("the player covers its card's frame as it is on screen", () => { + expect(playerBox({ left: 130, top: 260, width: 164, height: 70 }, { left: 100, top: 200, width: 800, height: 600 })) + .toEqual({ left: 30, top: 60, width: 164, height: 70 }); + }); }); diff --git a/www/README.md b/www/README.md index bfcfadde..92f94090 100644 --- a/www/README.md +++ b/www/README.md @@ -101,7 +101,8 @@ external images, including production optimizer URL validation. For a candidate: 2. Inspect browser Network and image `src`/`srcset`: external media and JSON must request the configured CDN directly, without `/_next/image?url=...` or a same-origin proxy. -3. Confirm no CORS errors and exercise video playback/seeking after host changes. +3. Confirm no CORS errors, that canvas video cards show their first frame (read from + the file, so videos need CORS too), and exercise video playback/seeking after host changes. These external asset bytes bypass Vercel optimization and delivery. HTML, JavaScript, local branding/landing media and generated social previews remain From b52e21a4defd297a28b9b3f0b48da6cd7e04d3bd Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 22:52:24 +0900 Subject: [PATCH 02/10] Play a card's video muted while the pointer rests on it Resting on a video's frame for 300 ms plays it over the card, muted and looping, without controls or focus; leaving stops it, and a click on it plays it on with sound and controls. Hovering never replaces a video played with sound, and needs a pointer that hovers and motion not reduced. The frame's checkered backdrop now belongs to the video, so the whole frame takes the pointer. The player is transparent and shows the first frame as its poster, so the backdrop shows around the video as it does around the first frame, and the first frame hides while the video plays over it. --- docs/guide/07-viewing.md | 7 +++-- spec/layout.md | 8 +++-- web/fx-web/src/canvas.css | 4 ++- web/fx-web/src/card.ts | 10 ++++--- web/fx-web/src/video.ts | 62 +++++++++++++++++++++++++++++++++++---- 5 files changed, 74 insertions(+), 17 deletions(-) diff --git a/docs/guide/07-viewing.md b/docs/guide/07-viewing.md index 548bae42..8e57ab5c 100644 --- a/docs/guide/07-viewing.md +++ b/docs/guide/07-viewing.md @@ -270,9 +270,10 @@ latest consistent snapshot. Images, text/JSON, audio, and video use generic previews; other file types can be opened in a new tab. Run canvas cards also show the first available recorded image output. A step without -one shows the first frame of its first MP4 or WebM video and a play button: the video -plays over the card, follows it as you pan and zoom, and stops when it ends, on Escape, -or when another card covers it. A count indicates multiple distinct available images +one shows the first frame of its first MP4 or WebM video. Resting the pointer on it +plays it muted and looping; click it, or its play button, to play it with sound. The +video plays over the card, follows it as you pan and zoom, and stops when it ends, on +Escape, or when another card covers it. A count indicates multiple distinct available images (or videos); select the card to inspect all outputs. The canvas uses the verified artifact inventory and never evaluates author code to produce thumbnails; a video's first frame is read in the browser once its card comes into view. Plan cards have no diff --git a/spec/layout.md b/spec/layout.md index 912df8a3..9ab25e63 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -257,6 +257,7 @@ including those that leave the deck. | Input | Closed deck | Expanded deck | |---|---|---| | Hover, only where `(hover: hover)` | raise the card under the pointer, and its linked partner; leaving restores the resting card | nothing; everything under the veil is inert | +| Rest on a card's video, only where `(hover: hover)` and motion is not reduced | after a moment it plays muted and looping over the card until the pointer leaves; a click on it plays it with sound and controls | the same | | Hover a bookmark row | raise that member, as hovering its card does | the camera glides to that member's card, which is outlined; leaving keeps the camera there | | Click a bookmark row | select that member (a frame member: its first step), which then rests in front | select it, and the camera glides to its card | | Still click on a card, a back card's visible rim or the badge | expand | select that member | @@ -274,9 +275,10 @@ including those that leave the deck. | A deck inside an expanded card | — | a click expands it one level deeper. Only the innermost level is interactive; the levels around it are its veil, so a click on them collapses one level and selects nothing | A video plays over its card's preview, following the card as the camera and the deck move it, and -selects nothing. It stops when it ends, on Esc (focus returns to its play control) and as soon as -anything covers its card: another card raised over it, its deck resting with another card in front, -or the veil of a deck it is not in. +selects nothing. Resting on it plays it muted, without taking focus or replacing a video played with +sound. It stops when it ends, when the pointer leaves a muted one, on Esc (focus returns to its play +control) and as soon as anything covers its card: another card raised over it, its deck resting with +another card in front, or the veil of a deck it is not in. On collapse, focus returns to the deck's front card. Raising a card never moves the focused element in the document. At most one deck per view is expanded, plus its nested levels. There are diff --git a/web/fx-web/src/canvas.css b/web/fx-web/src/canvas.css index 07b6bcbc..76c4fca5 100644 --- a/web/fx-web/src/canvas.css +++ b/web/fx-web/src/canvas.css @@ -56,8 +56,10 @@ .fx-canvas-play:focus-visible circle { stroke: #2563eb; stroke-width: 2px; } .fx-canvas-play svg { color: #fff; pointer-events: none; } .fx-canvas-video.is-playing .fx-canvas-play, .fx-canvas-video.is-unavailable .fx-canvas-play { display: none; } +.fx-canvas-video.is-playing .fx-canvas-node-image { visibility: hidden; } .fx-canvas-video.is-unavailable .fx-canvas-image-error { visibility: visible; } -.fx-canvas-player { position: absolute; box-sizing: border-box; border-radius: 6px; background: #18181b; object-fit: contain; } +.fx-canvas-player { position: absolute; box-sizing: border-box; border-radius: 6px; background: transparent; object-fit: contain; } +.fx-canvas-player.is-hover { cursor: pointer; } .fx-canvas-toolbar { position: absolute; bottom: 16px; left: 16px; display: flex; align-items: center; gap: 1px; padding: 3px; border: 1px solid #e4e4e7; border-radius: 9px; background: #fff; box-shadow: 0 2px 5px #00000008; } .fx-canvas-toolbar button { min-width: 32px; height: 30px; padding: 0 8px; border: 0; border-radius: 5px; background: #fff; color: #3f3f46; font: 13px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; cursor: pointer; } .fx-canvas-toolbar button:hover { background: #f4f4f5; } diff --git a/web/fx-web/src/card.ts b/web/fx-web/src/card.ts index 3b3f6efe..81145dbe 100644 --- a/web/fx-web/src/card.ts +++ b/web/fx-web/src/card.ts @@ -100,7 +100,8 @@ export function nodeCard(node: PositionedNode, canvasId: string, index: number, const clipId = `${canvasId}-preview-${index}`; const clip = element("clipPath", { id: clipId }); clip.append(element("rect", { ...frame, rx: 6 })); - group.append(clip, element("rect", { ...frame, rx: 6, fill: `url(#${canvasId}-checker)` })); + group.append(clip); + const backdrop = element("rect", { ...frame, rx: 6, fill: `url(#${canvasId}-checker)` }); const unavailable = preview.kind === "video" ? "Video unavailable" : "Image unavailable"; const image = element("image", { ...frame, preserveAspectRatio: "xMidYMid meet", "clip-path": `url(#${clipId})`, role: "img", "aria-label": preview.label, class: "fx-canvas-node-image" }); const fallback = element("text", { x: node.width / 2, y: frame.y + frame.height / 2, "text-anchor": "middle", "dominant-baseline": "middle", class: "fx-canvas-image-error", visibility: "hidden" }, unavailable); @@ -112,16 +113,17 @@ export function nodeCard(node: PositionedNode, canvasId: string, index: number, const border = element("rect", { ...frame, rx: 6, class: "fx-canvas-image-border" }); if (preview.kind === "image") { image.setAttribute("href", preview.url); - group.append(image, border, fallback); + group.append(backdrop, image, border, fallback); } else { // A video shows its first frame, which the canvas sets once the card is in view, and a play - // control; the canvas plays it over the frame (video.ts). + // control; the canvas plays it over the frame (video.ts). The backdrop takes the pointer, so + // resting anywhere on the frame plays it muted. const video = element("g", { class: "fx-canvas-video", "data-digest": preview.digest }); const play = element("g", { class: "fx-canvas-play", role: "button", tabindex: 0, "aria-label": `Play ${preview.label}` }); const center = { x: node.width / 2, y: frame.y + frame.height / 2 }; play.append(element("circle", { cx: center.x, cy: center.y, r: 18 }), createElement(Play, { x: center.x - 8, y: center.y - 8, width: 16, height: 16, "stroke-width": 2, "aria-hidden": "true" })); - video.append(image, border, fallback, play); + video.append(backdrop, image, border, fallback, play); group.append(video); } group.append(element("text", { x: node.width - 16, y: node.height - 22, "text-anchor": "end", class: "fx-canvas-image-count" }, count)); diff --git a/web/fx-web/src/video.ts b/web/fx-web/src/video.ts index f076fcf4..d9a60fb9 100644 --- a/web/fx-web/src/video.ts +++ b/web/fx-web/src/video.ts @@ -4,6 +4,8 @@ import type { CanvasPreview } from "./graph"; const STILL_WIDTH = 640; /** How many videos load at once for their first frames. */ const LOADS = 2; +/** How long the pointer rests on a video's frame before it plays there, muted, in milliseconds. */ +const HOVER_DELAY = 300; type Box = Pick; @@ -49,8 +51,9 @@ async function firstFrame(url: string, signal: AbortSignal): Promise { + if (!this.playing?.hover) return; + event.preventDefault(); + this.playing.hover = false; + this.mode(false); + this.player.focus({ preventScroll: true }); + }, { signal }); + this.player.addEventListener("pointerleave", () => { if (this.playing?.hover) this.stop(); }, { signal }); } /** The card whose video plays. */ @@ -121,6 +138,21 @@ export class CardVideos { }; play.addEventListener("click", start, { signal }); play.addEventListener("keydown", (event) => { if (event.key === "Enter" || event.key === " ") start(event); }, { signal }); + // Resting on the frame plays it muted, unless a video played with sound is on. + video.addEventListener("pointerenter", () => { + this.cancelHover(); + if (!this.canHover.matches || this.reducedMotion.matches || this.failed.has(preview.digest) || (this.playing && !this.playing.hover)) return; + this.hoverTimer = window.setTimeout(() => { + this.hoverTimer = null; + if (video.isConnected) this.play({ node, preview, card, video }, true); + }, HOVER_DELAY); + }, { signal }); + // The player appears under the pointer: moving onto it stays on the frame. + video.addEventListener("pointerleave", (event) => { + if (event.relatedTarget === this.player) return; + this.cancelHover(); + if (this.playing?.hover && this.playing.video === video) this.stop(); + }, { signal }); } /** After a redraw the player stays with its step while the step's card shows the same file. */ @@ -134,6 +166,7 @@ export class CardVideos { } stop() { + this.cancelHover(); const playing = this.playing; if (!playing) return; const focused = document.activeElement === this.player; @@ -157,17 +190,34 @@ export class CardVideos { this.ready.clear(); } - private play(playing: NonNullable) { + private play(playing: Omit, "hover">, hover = false) { this.stop(); - this.playing = playing; + this.playing = { ...playing, hover }; playing.video.classList.add("is-playing"); this.player.setAttribute("aria-label", playing.preview.label); + this.mode(hover); + // Its first frame shows until the video's own does; the card's backdrop shows around it. + const still = this.ready.get(playing.preview.digest); + if (still) this.player.poster = still; + else this.player.removeAttribute("poster"); this.player.src = playing.preview.url; this.player.hidden = false; this.follow(); - this.player.focus({ preventScroll: true }); + // Hovering never takes focus. + if (!hover) this.player.focus({ preventScroll: true }); void this.player.play().catch(() => {}); } + /** Muted, looping and bare while hovered; with sound and controls once asked for. */ + private mode(hover: boolean) { + this.player.muted = hover; + this.player.loop = hover; + this.player.controls = !hover; + this.player.classList.toggle("is-hover", hover); + } + private cancelHover() { + if (this.hoverTimer !== null) window.clearTimeout(this.hoverTimer); + this.hoverTimer = null; + } /** Keeps the player over its card's frame every frame, as the camera and the decks move it. */ private follow = () => { From c1edd69f05a12d41e15147dd1906a3faba41898d Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 23:03:08 +0900 Subject: [PATCH 03/10] Show the browser's video controls while a video plays on hover A hovered video now plays muted and looping with the native controls, as a played one does. A native video takes clicks for itself, so a click no longer turns on sound: pressing the hovered video, its controls included, keeps it playing after the pointer leaves, and its volume control unmutes it. --- docs/guide/07-viewing.md | 3 ++- spec/layout.md | 10 +++++----- web/fx-web/src/canvas.css | 1 - web/fx-web/src/video.ts | 25 +++++++++---------------- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/docs/guide/07-viewing.md b/docs/guide/07-viewing.md index 8e57ab5c..dfdd610c 100644 --- a/docs/guide/07-viewing.md +++ b/docs/guide/07-viewing.md @@ -271,7 +271,8 @@ audio, and video use generic previews; other file types can be opened in a new t Run canvas cards also show the first available recorded image output. A step without one shows the first frame of its first MP4 or WebM video. Resting the pointer on it -plays it muted and looping; click it, or its play button, to play it with sound. The +plays it muted and looping, with the browser's video controls; press it to keep it +playing after the pointer leaves, or use its play button to play it with sound. The video plays over the card, follows it as you pan and zoom, and stops when it ends, on Escape, or when another card covers it. A count indicates multiple distinct available images (or videos); select the card to inspect all outputs. The canvas uses the verified diff --git a/spec/layout.md b/spec/layout.md index 9ab25e63..bf8de816 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -257,7 +257,7 @@ including those that leave the deck. | Input | Closed deck | Expanded deck | |---|---|---| | Hover, only where `(hover: hover)` | raise the card under the pointer, and its linked partner; leaving restores the resting card | nothing; everything under the veil is inert | -| Rest on a card's video, only where `(hover: hover)` and motion is not reduced | after a moment it plays muted and looping over the card until the pointer leaves; a click on it plays it with sound and controls | the same | +| Rest on a card's video, only where `(hover: hover)` and motion is not reduced | after a moment it plays over the card, muted and looping, with its controls, until the pointer leaves; pressing it keeps it playing | the same | | Hover a bookmark row | raise that member, as hovering its card does | the camera glides to that member's card, which is outlined; leaving keeps the camera there | | Click a bookmark row | select that member (a frame member: its first step), which then rests in front | select it, and the camera glides to its card | | Still click on a card, a back card's visible rim or the badge | expand | select that member | @@ -275,10 +275,10 @@ including those that leave the deck. | A deck inside an expanded card | — | a click expands it one level deeper. Only the innermost level is interactive; the levels around it are its veil, so a click on them collapses one level and selects nothing | A video plays over its card's preview, following the card as the camera and the deck move it, and -selects nothing. Resting on it plays it muted, without taking focus or replacing a video played with -sound. It stops when it ends, when the pointer leaves a muted one, on Esc (focus returns to its play -control) and as soon as anything covers its card: another card raised over it, its deck resting with -another card in front, or the veil of a deck it is not in. +selects nothing. Resting on it plays it muted, without taking focus or replacing a video someone +played or pressed. It stops when it ends, when the pointer leaves one that is only hovered, on Esc +(focus returns to its play control) and as soon as anything covers its card: another card raised +over it, its deck resting with another card in front, or the veil of a deck it is not in. On collapse, focus returns to the deck's front card. Raising a card never moves the focused element in the document. At most one deck per view is expanded, plus its nested levels. There are diff --git a/web/fx-web/src/canvas.css b/web/fx-web/src/canvas.css index 76c4fca5..cd91efb7 100644 --- a/web/fx-web/src/canvas.css +++ b/web/fx-web/src/canvas.css @@ -59,7 +59,6 @@ .fx-canvas-video.is-playing .fx-canvas-node-image { visibility: hidden; } .fx-canvas-video.is-unavailable .fx-canvas-image-error { visibility: visible; } .fx-canvas-player { position: absolute; box-sizing: border-box; border-radius: 6px; background: transparent; object-fit: contain; } -.fx-canvas-player.is-hover { cursor: pointer; } .fx-canvas-toolbar { position: absolute; bottom: 16px; left: 16px; display: flex; align-items: center; gap: 1px; padding: 3px; border: 1px solid #e4e4e7; border-radius: 9px; background: #fff; box-shadow: 0 2px 5px #00000008; } .fx-canvas-toolbar button { min-width: 32px; height: 30px; padding: 0 8px; border: 0; border-radius: 5px; background: #fff; color: #3f3f46; font: 13px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; cursor: pointer; } .fx-canvas-toolbar button:hover { background: #f4f4f5; } diff --git a/web/fx-web/src/video.ts b/web/fx-web/src/video.ts index d9a60fb9..b81f0bcd 100644 --- a/web/fx-web/src/video.ts +++ b/web/fx-web/src/video.ts @@ -51,9 +51,9 @@ async function firstFrame(url: string, signal: AbortSignal): Promise { + // Leaving a hovered video stops it; pressing it, its controls included, keeps it playing. + this.player.addEventListener("pointerdown", () => { if (!this.playing?.hover) return; - event.preventDefault(); this.playing.hover = false; - this.mode(false); + this.player.loop = false; this.player.focus({ preventScroll: true }); }, { signal }); this.player.addEventListener("pointerleave", () => { if (this.playing?.hover) this.stop(); }, { signal }); @@ -195,7 +194,8 @@ export class CardVideos { this.playing = { ...playing, hover }; playing.video.classList.add("is-playing"); this.player.setAttribute("aria-label", playing.preview.label); - this.mode(hover); + this.player.muted = hover; + this.player.loop = hover; // Its first frame shows until the video's own does; the card's backdrop shows around it. const still = this.ready.get(playing.preview.digest); if (still) this.player.poster = still; @@ -207,13 +207,6 @@ export class CardVideos { if (!hover) this.player.focus({ preventScroll: true }); void this.player.play().catch(() => {}); } - /** Muted, looping and bare while hovered; with sound and controls once asked for. */ - private mode(hover: boolean) { - this.player.muted = hover; - this.player.loop = hover; - this.player.controls = !hover; - this.player.classList.toggle("is-hover", hover); - } private cancelHover() { if (this.hoverTimer !== null) window.clearTimeout(this.hoverTimer); this.hoverTimer = null; From 265c129dcae9029d6030124c4b88dc442616c59e Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 23:11:06 +0900 Subject: [PATCH 04/10] Give the canvas player its own controls Native video controls are sized for a page, not a zoomable canvas: they crowd a card seen from afar and do not match the canvas. The player now has its own (video-controls.ts): play, a seek slider showing what has played and loaded, time, mute and full screen. They keep their size on screen at any zoom, drop the time and then seek and full screen as the player narrows, and fade while it plays once the pointer rests. With focus in the player, Space or K plays and pauses, the arrow keys seek 5 seconds, M mutes, F goes full screen and Esc closes it; its buttons and slider keep their own keys. A press on the picture of a hovered video keeps it playing with sound, as pressing its controls keeps it playing. Play and pause are filled glyphs of our own, on the card too: Lucide's outlined ones read poorly on a video. The details panel keeps the browser's controls. --- docs/guide/07-viewing.md | 9 +- spec/layout.md | 8 +- web/fx-web/src/canvas.css | 24 ++- web/fx-web/src/canvas.ts | 4 +- web/fx-web/src/card.ts | 4 +- web/fx-web/src/video-controls.ts | 200 ++++++++++++++++++++++++ web/fx-web/src/video.ts | 81 +++++----- web/fx-web/tests/video-controls.test.ts | 22 +++ 8 files changed, 303 insertions(+), 49 deletions(-) create mode 100644 web/fx-web/src/video-controls.ts create mode 100644 web/fx-web/tests/video-controls.test.ts diff --git a/docs/guide/07-viewing.md b/docs/guide/07-viewing.md index dfdd610c..881dd957 100644 --- a/docs/guide/07-viewing.md +++ b/docs/guide/07-viewing.md @@ -271,10 +271,11 @@ audio, and video use generic previews; other file types can be opened in a new t Run canvas cards also show the first available recorded image output. A step without one shows the first frame of its first MP4 or WebM video. Resting the pointer on it -plays it muted and looping, with the browser's video controls; press it to keep it -playing after the pointer leaves, or use its play button to play it with sound. The -video plays over the card, follows it as you pan and zoom, and stops when it ends, on -Escape, or when another card covers it. A count indicates multiple distinct available images +plays it muted and looping; click it to keep it playing with sound, or use its play +button. The video plays over the card, follows it as you pan and zoom, and stops when it +ends, on Escape, or when another card covers it. Its controls keep their size at any +zoom (play, seek, time, mute and full screen, fewer on a small card); with the player +focused, Space or K plays and pauses, the arrow keys seek, M mutes and F goes full screen. A count indicates multiple distinct available images (or videos); select the card to inspect all outputs. The canvas uses the verified artifact inventory and never evaluates author code to produce thumbnails; a video's first frame is read in the browser once its card comes into view. Plan cards have no diff --git a/spec/layout.md b/spec/layout.md index bf8de816..93434e77 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -257,7 +257,7 @@ including those that leave the deck. | Input | Closed deck | Expanded deck | |---|---|---| | Hover, only where `(hover: hover)` | raise the card under the pointer, and its linked partner; leaving restores the resting card | nothing; everything under the veil is inert | -| Rest on a card's video, only where `(hover: hover)` and motion is not reduced | after a moment it plays over the card, muted and looping, with its controls, until the pointer leaves; pressing it keeps it playing | the same | +| Rest on a card's video, only where `(hover: hover)` and motion is not reduced | after a moment it plays over the card, muted and looping, with its controls, until the pointer leaves; pressing it keeps it playing, and a press on its picture turns its sound on | the same | | Hover a bookmark row | raise that member, as hovering its card does | the camera glides to that member's card, which is outlined; leaving keeps the camera there | | Click a bookmark row | select that member (a frame member: its first step), which then rests in front | select it, and the camera glides to its card | | Still click on a card, a back card's visible rim or the badge | expand | select that member | @@ -278,7 +278,11 @@ A video plays over its card's preview, following the card as the camera and the selects nothing. Resting on it plays it muted, without taking focus or replacing a video someone played or pressed. It stops when it ends, when the pointer leaves one that is only hovered, on Esc (focus returns to its play control) and as soon as anything covers its card: another card raised -over it, its deck resting with another card in front, or the veil of a deck it is not in. +over it, its deck resting with another card in front, or the veil of a deck it is not in. Its +controls are the viewer's own and keep their size on screen at any zoom: play, seek, time, mute and +full screen, the time going first and then seek and full screen as the player gets narrow. While it +plays they fade once the pointer rests. With focus in the player, Space or K plays and pauses, the +arrow keys seek 5 seconds, M mutes and F goes full screen. On collapse, focus returns to the deck's front card. Raising a card never moves the focused element in the document. At most one deck per view is expanded, plus its nested levels. There are diff --git a/web/fx-web/src/canvas.css b/web/fx-web/src/canvas.css index cd91efb7..eb036cab 100644 --- a/web/fx-web/src/canvas.css +++ b/web/fx-web/src/canvas.css @@ -58,7 +58,29 @@ .fx-canvas-video.is-playing .fx-canvas-play, .fx-canvas-video.is-unavailable .fx-canvas-play { display: none; } .fx-canvas-video.is-playing .fx-canvas-node-image { visibility: hidden; } .fx-canvas-video.is-unavailable .fx-canvas-image-error { visibility: visible; } -.fx-canvas-player { position: absolute; box-sizing: border-box; border-radius: 6px; background: transparent; object-fit: contain; } +.fx-canvas-player { position: absolute; box-sizing: border-box; overflow: hidden; border-radius: 6px; outline: none; container-type: inline-size; font: 11px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; } +.fx-canvas-player:focus-visible { box-shadow: 0 0 0 2px #2563eb; } +.fx-canvas-player > video { display: block; width: 100%; height: 100%; object-fit: contain; } +.fx-canvas-player:fullscreen { border-radius: 0; background: #000; } +.fx-canvas-player.is-idle { cursor: none; } +/* Its controls (video-controls.ts) keep their screen size at any zoom; a small player keeps fewer. */ +.fx-video-controls { position: absolute; right: 0; bottom: 0; left: 0; display: flex; align-items: center; gap: 2px; padding: 14px 4px 3px; background: linear-gradient(#0000, #0000008c); color: #fff; transition: opacity .2s; } +.fx-canvas-player.is-idle .fx-video-controls { opacity: 0; } +.fx-video-controls button { display: grid; flex: none; place-items: center; width: 24px; height: 24px; padding: 0; border: 0; border-radius: 5px; background: none; color: inherit; cursor: pointer; } +.fx-video-controls button[hidden] { display: none; } +.fx-video-controls button:hover { background: #ffffff2e; } +.fx-video-controls button:focus-visible { outline: 2px solid #93c5fd; outline-offset: -2px; } +.fx-video-time { flex: none; padding: 0 3px; font-variant-numeric: tabular-nums; white-space: nowrap; } +.fx-video-seek { flex: 1; min-width: 0; height: 16px; margin: 0 4px; background: none; cursor: pointer; appearance: none; } +.fx-video-seek:disabled { cursor: default; } +.fx-video-seek:focus-visible { outline: 2px solid #93c5fd; outline-offset: 1px; border-radius: 2px; } +.fx-video-seek::-webkit-slider-runnable-track { height: 3px; border-radius: 2px; background: linear-gradient(to right, #fff var(--fx-played, 0%), #ffffff80 var(--fx-played, 0%) var(--fx-buffered, 0%), #ffffff40 var(--fx-buffered, 0%)); } +.fx-video-seek::-moz-range-track { height: 3px; border-radius: 2px; background: linear-gradient(to right, #fff var(--fx-played, 0%), #ffffff80 var(--fx-played, 0%) var(--fx-buffered, 0%), #ffffff40 var(--fx-buffered, 0%)); } +.fx-video-seek::-webkit-slider-thumb { width: 10px; height: 10px; margin-top: -3.5px; border: 0; border-radius: 50%; background: #fff; appearance: none; } +.fx-video-seek::-moz-range-thumb { width: 10px; height: 10px; border: 0; border-radius: 50%; background: #fff; } +@container (max-width: 220px) { .fx-video-time { display: none; } } +@container (max-width: 150px) { .fx-video-seek, .fx-video-controls .fx-video-fullscreen { display: none; } .fx-video-controls { justify-content: space-between; } } +@container (max-width: 64px) { .fx-video-controls { display: none; } } .fx-canvas-toolbar { position: absolute; bottom: 16px; left: 16px; display: flex; align-items: center; gap: 1px; padding: 3px; border: 1px solid #e4e4e7; border-radius: 9px; background: #fff; box-shadow: 0 2px 5px #00000008; } .fx-canvas-toolbar button { min-width: 32px; height: 30px; padding: 0 8px; border: 0; border-radius: 5px; background: #fff; color: #3f3f46; font: 13px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; cursor: pointer; } .fx-canvas-toolbar button:hover { background: #f4f4f5; } diff --git a/web/fx-web/src/canvas.ts b/web/fx-web/src/canvas.ts index b1646e74..137cb6f2 100644 --- a/web/fx-web/src/canvas.ts +++ b/web/fx-web/src/canvas.ts @@ -170,7 +170,7 @@ export class CanvasController { this.svg.addEventListener("pointerup", this.onPointerUp, { signal: this.abort.signal }); this.svg.addEventListener("pointercancel", this.onPointerUp, { signal: this.abort.signal }); // The pointer moving onto the player stays on its card. - this.svg.addEventListener("pointerleave", (event) => { if (event.relatedTarget !== this.videos.player) this.hover(null); }, { signal: this.abort.signal }); + this.svg.addEventListener("pointerleave", (event) => { if (!this.videos.player.contains(event.relatedTarget as Node | null)) this.hover(null); }, { signal: this.abort.signal }); // A deck card is the expand target: the capture phase runs before a card's own selection. this.svg.addEventListener("click", this.onDeckClick, { capture: true, signal: this.abort.signal }); // Raising a card moves focus out of the canvas, so Esc listens on the document while open. @@ -178,7 +178,7 @@ export class CanvasController { if (event.key !== "Escape" || !this.open || event.defaultPrevented) return; const target = event.target as HTMLElement | null; // A field keeps its own keys; Esc on the card's player closes the player first. - if (target?.closest("input, textarea, select, video, [contenteditable]")) return; + if (target?.closest("input, textarea, select, [contenteditable], .fx-canvas-player")) return; event.preventDefault(); this.collapse(); }, { capture: true, signal: this.abort.signal }); diff --git a/web/fx-web/src/card.ts b/web/fx-web/src/card.ts index 81145dbe..557ccdbc 100644 --- a/web/fx-web/src/card.ts +++ b/web/fx-web/src/card.ts @@ -1,8 +1,8 @@ -import { createElement, Play } from "lucide"; import { displayPortName } from "./ports"; import { canvasNodeGeometry, canvasPreviewReserved, PORT_ROW, PORT_TOP, type PositionedNode } from "./layout"; import { stateIcon, stateIconParts } from "./state-icon"; import type { CanvasStep } from "./graph"; +import { mediaGlyph } from "./video-controls"; const SVG = "http://www.w3.org/2000/svg"; @@ -122,7 +122,7 @@ export function nodeCard(node: PositionedNode, canvasId: string, index: number, const play = element("g", { class: "fx-canvas-play", role: "button", tabindex: 0, "aria-label": `Play ${preview.label}` }); const center = { x: node.width / 2, y: frame.y + frame.height / 2 }; play.append(element("circle", { cx: center.x, cy: center.y, r: 18 }), - createElement(Play, { x: center.x - 8, y: center.y - 8, width: 16, height: 16, "stroke-width": 2, "aria-hidden": "true" })); + mediaGlyph("play", { x: center.x - 8, y: center.y - 8, width: 16, height: 16 })); video.append(backdrop, image, border, fallback, play); group.append(video); } diff --git a/web/fx-web/src/video-controls.ts b/web/fx-web/src/video-controls.ts new file mode 100644 index 00000000..e837639c --- /dev/null +++ b/web/fx-web/src/video-controls.ts @@ -0,0 +1,200 @@ +import { createElement, Maximize, Minimize, Volume2, VolumeX, type IconNode } from "lucide"; + +/** How long the pointer rests before a playing video's controls fade, in milliseconds. */ +const IDLE_DELAY = 2000; +/** How far the arrow keys seek, in seconds. */ +const SEEK_STEP = 5; + +/** + * Filled play and pause glyphs, on a 24-unit square, drawn in the current colour. Unlike the rest + * of the viewer's chrome they are not Lucide's: its outlined ones read poorly on a video. + */ +export function mediaGlyph(kind: "play" | "pause", attributes: Record = {}) { + const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); + for (const [name, value] of Object.entries({ viewBox: "0 0 24 24", fill: "currentColor", "aria-hidden": "true", ...attributes })) svg.setAttribute(name, String(value)); + if (kind === "play") { + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute("d", "M7 4.5v15a1 1 0 0 0 1.53.85l12-7.5a1 1 0 0 0 0-1.7l-12-7.5A1 1 0 0 0 7 4.5z"); + svg.append(path); + } else for (const x of [6, 14]) { + const bar = document.createElementNS("http://www.w3.org/2000/svg", "rect"); + for (const [name, value] of Object.entries({ x, y: 5, width: 4, height: 14, rx: 1.25 })) bar.setAttribute(name, String(value)); + svg.append(bar); + } + return svg; +} + +/** A video time as players show it: "0:07", "12:30", "1:02:03". */ +export function videoTime(seconds: number) { + const whole = Number.isFinite(seconds) ? Math.max(0, Math.floor(seconds)) : 0; + const [hours, minutes, rest] = [Math.floor(whole / 3600), Math.floor(whole / 60) % 60, whole % 60]; + const two = (value: number) => String(value).padStart(2, "0"); + return hours ? `${hours}:${two(minutes)}:${two(rest)}` : `${minutes}:${two(rest)}`; +} + +/** Where a seek by `by` seconds lands, within the video. */ +export function seekTarget(at: number, by: number, duration: number) { + const end = Number.isFinite(duration) ? duration : at + Math.max(0, by); + return Math.min(end, Math.max(0, at + by)); +} + +/** What a key does to a focused player; keys on its own buttons and slider stay theirs. */ +export function playerKey(key: string, target: "button" | "slider" | "other"): "toggle" | "back" | "forward" | "mute" | "fullscreen" | "close" | null { + if (key === "Escape") return "close"; + if ((key === " " || key === "Enter") && target === "button") return null; + if ((key === "ArrowLeft" || key === "ArrowRight") && target === "slider") return null; + switch (key.toLowerCase()) { + case " ": case "k": return "toggle"; + case "arrowleft": return "back"; + case "arrowright": return "forward"; + case "m": return "mute"; + case "f": return "fullscreen"; + default: return null; + } +} + +/** + * A video with the viewer's own controls: play, a seek slider, time, mute and full screen, sized in + * screen pixels and fewer when the player is small on screen, so they stay usable at any zoom of + * the canvas under it. While it plays they fade when the pointer rests; a press anywhere shows them. + * `press` sees every press first: it returns true to take a press on the picture for itself. + */ +export class VideoControls { + readonly element = document.createElement("div"); + readonly video = document.createElement("video"); + private readonly bar = document.createElement("div"); + private readonly playButton = this.button("fx-video-play"); + private readonly seek = document.createElement("input"); + private readonly time = document.createElement("span"); + private readonly muteButton = this.button("fx-video-mute"); + private readonly fullscreenButton = this.button("fx-video-fullscreen"); + private idle: number | null = null; + private scrubbing = false; + + constructor(signal: AbortSignal, close: () => void, press: (picture: boolean) => boolean) { + const { element, video, bar, seek } = this; + element.className = "fx-canvas-player"; + element.tabIndex = -1; + element.setAttribute("role", "group"); + element.hidden = true; + video.playsInline = true; + bar.className = "fx-video-controls"; + seek.type = "range"; + seek.className = "fx-video-seek"; + seek.min = "0"; + seek.step = "any"; + seek.setAttribute("aria-label", "Seek"); + this.time.className = "fx-video-time"; + this.fullscreenButton.hidden = !document.fullscreenEnabled && !("webkitEnterFullscreen" in video); + bar.append(this.playButton, seek, this.time, this.muteButton, this.fullscreenButton); + element.append(video, bar); + + const on = (type: K, listener: () => void) => video.addEventListener(type, listener, { signal }); + for (const type of ["play", "pause", "timeupdate", "durationchange", "progress", "volumechange", "emptied"] as const) on(type, () => this.render()); + on("play", () => this.wake()); + on("pause", () => this.wake()); + document.addEventListener("fullscreenchange", () => this.render(), { signal }); + + this.playButton.addEventListener("click", () => this.toggle(), { signal }); + this.muteButton.addEventListener("click", () => { video.muted = !video.muted; }, { signal }); + this.fullscreenButton.addEventListener("click", () => this.fullscreen(), { signal }); + seek.addEventListener("input", () => { video.currentTime = Number(seek.value); this.render(); }, { signal }); + seek.addEventListener("pointerdown", () => { this.scrubbing = true; }, { signal }); + for (const type of ["pointerup", "pointercancel", "change"] as const) seek.addEventListener(type, () => { this.scrubbing = false; }, { signal }); + + // The picture plays and pauses, unless `press` took the press. + let taken = false; + element.addEventListener("pointerdown", (event) => { taken = press(event.target === video); this.wake(); }, { signal }); + video.addEventListener("click", () => { if (taken) taken = false; else this.toggle(); }, { signal }); + element.addEventListener("pointermove", () => this.wake(), { signal }); + element.addEventListener("pointerleave", () => { if (!video.paused) this.rest(); }, { signal }); + element.addEventListener("focusin", () => this.wake(), { signal }); + element.addEventListener("keydown", (event) => { + const target = event.target === seek ? "slider" : (event.target as Element).closest("button") ? "button" : "other"; + const action = playerKey(event.key, target); + if (!action || event.defaultPrevented || event.metaKey || event.ctrlKey || event.altKey) return; + event.preventDefault(); + event.stopPropagation(); + this.wake(); + if (action === "close") close(); + else if (action === "toggle") this.toggle(); + else if (action === "mute") video.muted = !video.muted; + else if (action === "fullscreen") this.fullscreen(); + else video.currentTime = seekTarget(video.currentTime, action === "back" ? -SEEK_STEP : SEEK_STEP, video.duration); + }, { signal }); + signal.addEventListener("abort", () => this.clearIdle(), { once: true }); + this.render(); + } + + /** Whether the player fills the screen: the canvas then leaves its size to the browser. */ + get fullscreened() { return document.fullscreenElement === this.element; } + + /** Before another video: controls shown, nothing pending. */ + reset() { + this.clearIdle(); + this.scrubbing = false; + this.element.classList.remove("is-idle"); + if (this.fullscreened) void document.exitFullscreen().catch(() => {}); + } + + private toggle() { + if (this.video.paused) void this.video.play().catch(() => {}); + else this.video.pause(); + } + private fullscreen() { + if (this.fullscreened) void document.exitFullscreen().catch(() => {}); + else if (document.fullscreenEnabled) void this.element.requestFullscreen().catch(() => {}); + // iOS shows a video full screen only in its own player. + else (this.video as HTMLVideoElement & { webkitEnterFullscreen?: () => void }).webkitEnterFullscreen?.(); + } + + /** Shows the controls; while the video plays they fade again once the pointer rests. */ + private wake() { + this.clearIdle(); + this.element.classList.remove("is-idle"); + if (!this.video.paused) this.idle = window.setTimeout(() => this.rest(), IDLE_DELAY); + } + private rest() { + this.clearIdle(); + if (!this.video.paused && !this.bar.contains(document.activeElement)) this.element.classList.add("is-idle"); + } + private clearIdle() { + if (this.idle !== null) window.clearTimeout(this.idle); + this.idle = null; + } + + private render() { + const { video, seek } = this; + const duration = Number.isFinite(video.duration) ? video.duration : 0; + const playing = !video.paused; + this.icon(this.playButton, () => mediaGlyph(playing ? "pause" : "play", { width: 15, height: 15 }), playing ? "Pause" : "Play"); + this.icon(this.muteButton, lucide(video.muted ? VolumeX : Volume2), video.muted ? "Unmute" : "Mute"); + this.icon(this.fullscreenButton, lucide(this.fullscreened ? Minimize : Maximize), this.fullscreened ? "Exit full screen" : "Full screen"); + seek.max = String(duration); + seek.disabled = duration === 0; + if (!this.scrubbing) seek.value = String(video.currentTime); + const share = (at: number) => `${duration ? Math.min(100, (at / duration) * 100) : 0}%`; + const buffered = video.buffered.length ? video.buffered.end(video.buffered.length - 1) : 0; + seek.style.setProperty("--fx-played", share(video.currentTime)); + seek.style.setProperty("--fx-buffered", share(buffered)); + const text = `${videoTime(video.currentTime)} / ${videoTime(duration)}`; + seek.setAttribute("aria-valuetext", `${videoTime(video.currentTime)} of ${videoTime(duration)}`); + if (this.time.textContent !== text) this.time.textContent = text; + } + + private button(className: string) { + const button = document.createElement("button"); + button.type = "button"; + button.className = className; + return button; + } + /** Swaps a button's icon and name only when they change. */ + private icon(button: HTMLButtonElement, icon: () => SVGElement, label: string) { + if (button.getAttribute("aria-label") === label) return; + button.setAttribute("aria-label", label); + button.title = label; + button.replaceChildren(icon()); + } +} + +const lucide = (icon: IconNode) => () => createElement(icon, { width: 15, height: 15, "stroke-width": 2, "aria-hidden": "true" }); diff --git a/web/fx-web/src/video.ts b/web/fx-web/src/video.ts index b81f0bcd..fd68328b 100644 --- a/web/fx-web/src/video.ts +++ b/web/fx-web/src/video.ts @@ -1,4 +1,5 @@ import type { CanvasPreview } from "./graph"; +import { VideoControls } from "./video-controls"; /** The widest first frame kept, in pixels: a card's frame is 328 canvas units wide. */ const STILL_WIDTH = 640; @@ -51,9 +52,9 @@ async function firstFrame(url: string, signal: AbortSignal): Promise void)[] = []; private loading = 0; private readonly abort = new AbortController(); - readonly player = document.createElement("video"); + private readonly controls: VideoControls; + /** The player: the video and its controls, laid over the playing card's frame. */ + readonly player: HTMLDivElement; + private readonly media: HTMLVideoElement; /** The video playing, and whether it plays only while hovered (muted, looping, until the pointer leaves). */ private playing: { node: string; preview: CanvasPreview; card: SVGGElement; video: SVGGElement; hover: boolean } | null = null; private frame: number | null = null; @@ -85,30 +89,25 @@ export class CardVideos { constructor(private readonly host: HTMLElement) { const signal = this.abort.signal; - this.player.className = "fx-canvas-player"; - this.player.controls = true; - this.player.playsInline = true; - this.player.hidden = true; - this.player.addEventListener("ended", () => this.stop(), { signal }); - this.player.addEventListener("error", () => { - if (!this.playing || !this.player.getAttribute("src")) return; + // Leaving a hovered video stops it. Pressing it keeps it playing, and a press on its picture + // turns its sound on rather than pausing it. + this.controls = new VideoControls(signal, () => this.stop(), (picture) => { + if (!this.playing?.hover) return false; + this.playing.hover = false; + this.media.loop = false; + if (picture) this.media.muted = false; + this.player.focus({ preventScroll: true }); + return picture; + }); + this.player = this.controls.element; + this.media = this.controls.video; + this.media.addEventListener("ended", () => this.stop(), { signal }); + this.media.addEventListener("error", () => { + if (!this.playing || !this.media.getAttribute("src")) return; this.failed.add(this.playing.preview.digest); this.playing.video.classList.add("is-unavailable"); this.stop(); }, { signal }); - this.player.addEventListener("keydown", (event) => { - if (event.key !== "Escape" || event.defaultPrevented) return; - event.preventDefault(); - event.stopPropagation(); - this.stop(); - }, { signal }); - // Leaving a hovered video stops it; pressing it, its controls included, keeps it playing. - this.player.addEventListener("pointerdown", () => { - if (!this.playing?.hover) return; - this.playing.hover = false; - this.player.loop = false; - this.player.focus({ preventScroll: true }); - }, { signal }); this.player.addEventListener("pointerleave", () => { if (this.playing?.hover) this.stop(); }, { signal }); } @@ -148,7 +147,7 @@ export class CardVideos { }, { signal }); // The player appears under the pointer: moving onto it stays on the frame. video.addEventListener("pointerleave", (event) => { - if (event.relatedTarget === this.player) return; + if (this.player.contains(event.relatedTarget as Node | null)) return; this.cancelHover(); if (this.playing?.hover && this.playing.video === video) this.stop(); }, { signal }); @@ -168,15 +167,16 @@ export class CardVideos { this.cancelHover(); const playing = this.playing; if (!playing) return; - const focused = document.activeElement === this.player; + const focused = this.player.contains(document.activeElement); this.playing = null; if (this.frame !== null) cancelAnimationFrame(this.frame); this.frame = null; playing.video.classList.remove("is-playing"); - this.player.pause(); + this.controls.reset(); + this.media.pause(); this.player.hidden = true; - this.player.removeAttribute("src"); - this.player.load(); + this.media.removeAttribute("src"); + this.media.load(); if (focused) playing.video.querySelector(".fx-canvas-play")?.focus({ preventScroll: true }); } @@ -194,30 +194,35 @@ export class CardVideos { this.playing = { ...playing, hover }; playing.video.classList.add("is-playing"); this.player.setAttribute("aria-label", playing.preview.label); - this.player.muted = hover; - this.player.loop = hover; + this.media.muted = hover; + this.media.loop = hover; // Its first frame shows until the video's own does; the card's backdrop shows around it. const still = this.ready.get(playing.preview.digest); - if (still) this.player.poster = still; - else this.player.removeAttribute("poster"); - this.player.src = playing.preview.url; + if (still) this.media.poster = still; + else this.media.removeAttribute("poster"); + this.media.src = playing.preview.url; this.player.hidden = false; this.follow(); // Hovering never takes focus. if (!hover) this.player.focus({ preventScroll: true }); - void this.player.play().catch(() => {}); + void this.media.play().catch(() => {}); } private cancelHover() { if (this.hoverTimer !== null) window.clearTimeout(this.hoverTimer); this.hoverTimer = null; } - /** Keeps the player over its card's frame every frame, as the camera and the decks move it. */ + /** + * Keeps the player over its card's frame every frame, as the camera and the decks move it; full + * screen, the browser places it. + */ private follow = () => { const frame = this.playing?.video.querySelector(".fx-canvas-image-border"); if (!frame) return; - const box = playerBox(frame.getBoundingClientRect(), this.host.getBoundingClientRect()); - for (const side of ["left", "top", "width", "height"] as const) this.player.style[side] = `${box[side]}px`; + if (!this.controls.fullscreened) { + const box = playerBox(frame.getBoundingClientRect(), this.host.getBoundingClientRect()); + for (const side of ["left", "top", "width", "height"] as const) this.player.style[side] = `${box[side]}px`; + } this.frame = requestAnimationFrame(this.follow); }; diff --git a/web/fx-web/tests/video-controls.test.ts b/web/fx-web/tests/video-controls.test.ts new file mode 100644 index 00000000..c197ef05 --- /dev/null +++ b/web/fx-web/tests/video-controls.test.ts @@ -0,0 +1,22 @@ +import { expect, test } from "bun:test"; +import { playerKey, seekTarget, videoTime } from "../src/video-controls"; + +test("video times read as players show them, and an unknown length as zero", () => { + expect([0, 7.9, 64, 750, 3723, -3, NaN, Infinity].map(videoTime)) + .toEqual(["0:00", "0:07", "1:04", "12:30", "1:02:03", "0:00", "0:00", "0:00"]); +}); + +test("seeking stays within the video", () => { + expect(seekTarget(3, -5, 8)).toBe(0); + expect(seekTarget(6, 5, 8)).toBe(8); + expect(seekTarget(2, 5, 8)).toBe(7); + // A stream of unknown length seeks forward freely. + expect(seekTarget(2, 5, Infinity)).toBe(7); +}); + +test("player keys leave the player's own buttons and slider their keys", () => { + expect(["Escape", " ", "k", "K", "m", "f", "ArrowLeft", "ArrowRight", "x"].map((key) => playerKey(key, "other"))) + .toEqual(["close", "toggle", "toggle", "toggle", "mute", "fullscreen", "back", "forward", null]); + expect([" ", "Enter", "Escape"].map((key) => playerKey(key, "button"))).toEqual([null, null, "close"]); + expect(["ArrowLeft", "ArrowRight", " "].map((key) => playerKey(key, "slider"))).toEqual([null, null, "toggle"]); +}); From 99e90cc78bef55cb32de4bd037849359a3fac3fc Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 23:12:38 +0900 Subject: [PATCH 05/10] Continue a card's video where it stopped Hovering a video again, or playing it, now continues from where it last stopped (pointer left, Esc, covered) rather than from the start; one that played to its end starts over. The player remembers each file's place for as long as the canvas lives, and skips the first-frame poster when it continues, so the start does not flash before the seek lands. --- docs/guide/07-viewing.md | 3 ++- spec/layout.md | 3 ++- web/fx-web/src/video.ts | 29 ++++++++++++++++++++++++++--- web/fx-web/tests/previews.test.ts | 9 ++++++++- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/docs/guide/07-viewing.md b/docs/guide/07-viewing.md index 881dd957..04e76b5e 100644 --- a/docs/guide/07-viewing.md +++ b/docs/guide/07-viewing.md @@ -273,7 +273,8 @@ Run canvas cards also show the first available recorded image output. A step wit one shows the first frame of its first MP4 or WebM video. Resting the pointer on it plays it muted and looping; click it to keep it playing with sound, or use its play button. The video plays over the card, follows it as you pan and zoom, and stops when it -ends, on Escape, or when another card covers it. Its controls keep their size at any +ends, on Escape, or when another card covers it; played again, it continues where it +stopped. Its controls keep their size at any zoom (play, seek, time, mute and full screen, fewer on a small card); with the player focused, Space or K plays and pauses, the arrow keys seek, M mutes and F goes full screen. A count indicates multiple distinct available images (or videos); select the card to inspect all outputs. The canvas uses the verified diff --git a/spec/layout.md b/spec/layout.md index 93434e77..6fe4b8c8 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -278,7 +278,8 @@ A video plays over its card's preview, following the card as the camera and the selects nothing. Resting on it plays it muted, without taking focus or replacing a video someone played or pressed. It stops when it ends, when the pointer leaves one that is only hovered, on Esc (focus returns to its play control) and as soon as anything covers its card: another card raised -over it, its deck resting with another card in front, or the veil of a deck it is not in. Its +over it, its deck resting with another card in front, or the veil of a deck it is not in. Resting on +it or playing it again continues where it stopped, unless it played to its end. Its controls are the viewer's own and keep their size on screen at any zoom: play, seek, time, mute and full screen, the time going first and then seek and full screen as the player gets narrow. While it plays they fade once the pointer rests. With focus in the player, Space or K plays and pauses, the diff --git a/web/fx-web/src/video.ts b/web/fx-web/src/video.ts index fd68328b..f394ba88 100644 --- a/web/fx-web/src/video.ts +++ b/web/fx-web/src/video.ts @@ -10,6 +10,12 @@ const HOVER_DELAY = 300; type Box = Pick; +/** Where a video stopped at `time` plays from next: there, or from the start once it has played through. */ +export function resumeAt(time: number, duration: number): number | null { + if (!Number.isFinite(time) || time <= 0) return null; + return Number.isFinite(duration) && time >= duration - 0.25 ? null : time; +} + /** Where the player goes, in the host's pixels: over the frame as it is on screen now. */ export function playerBox(frame: Box, host: Box) { return { left: frame.left - host.left, top: frame.top - host.top, width: frame.width, height: frame.height }; @@ -62,6 +68,10 @@ export class CardVideos { private readonly ready = new Map(); /** Files the player could not play: their cards say so and offer no play control. */ private readonly failed = new Set(); + /** Where each file stopped short of its end: hovering or playing it again continues there. */ + private readonly positions = new Map(); + /** What only the current video listens for; a stop ends it. */ + private session = new AbortController(); private readonly waiting = new Map void>(); private readonly observer = new IntersectionObserver((entries) => { for (const entry of entries) { @@ -169,6 +179,10 @@ export class CardVideos { if (!playing) return; const focused = this.player.contains(document.activeElement); this.playing = null; + this.session.abort(); + const at = resumeAt(this.media.currentTime, this.media.duration); + if (at === null) this.positions.delete(playing.preview.digest); + else this.positions.set(playing.preview.digest, at); if (this.frame !== null) cancelAnimationFrame(this.frame); this.frame = null; playing.video.classList.remove("is-playing"); @@ -196,11 +210,20 @@ export class CardVideos { this.player.setAttribute("aria-label", playing.preview.label); this.media.muted = hover; this.media.loop = hover; - // Its first frame shows until the video's own does; the card's backdrop shows around it. - const still = this.ready.get(playing.preview.digest); + this.session = new AbortController(); + // A video it stopped short of its end continues there. One starting over shows its first frame + // until the video's own does; the card's backdrop shows around it. + const { digest, url } = playing.preview; + const at = this.positions.get(digest); + const still = at === undefined ? this.ready.get(digest) : undefined; if (still) this.media.poster = still; else this.media.removeAttribute("poster"); - this.media.src = playing.preview.url; + this.media.src = url; + if (at !== undefined) { + this.media.currentTime = at; + // Set before the video has loaded, the time is where it starts; a browser that drops it seeks once it can. + this.media.addEventListener("loadedmetadata", () => { if (this.media.currentTime < at - 0.5) this.media.currentTime = at; }, { once: true, signal: this.session.signal }); + } this.player.hidden = false; this.follow(); // Hovering never takes focus. diff --git a/web/fx-web/tests/previews.test.ts b/web/fx-web/tests/previews.test.ts index 9495632d..86314264 100644 --- a/web/fx-web/tests/previews.test.ts +++ b/web/fx-web/tests/previews.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { canvasGraph, type Artifact, type ViewerRun } from "../src/index"; -import { playerBox } from "../src/video"; +import { playerBox, resumeAt } from "../src/video"; import { graph } from "./graph.test"; const sourceDigest = "a".repeat(64); @@ -120,6 +120,13 @@ describe("recorded canvas previews", () => { expect(canvasGraph(recorded).nodes[0].preview).toMatchObject({ kind: "image", digest: outputDigest, count: 1 }); }); + test("a video continues where it stopped, and starts over once it has played through", () => { + expect(resumeAt(3.2, 8)).toBe(3.2); + expect([resumeAt(0, 8), resumeAt(7.9, 8), resumeAt(8, 8), resumeAt(NaN, 8)]).toEqual([null, null, null, null]); + // A stream of unknown length always continues. + expect(resumeAt(40, Infinity)).toBe(40); + }); + test("the player covers its card's frame as it is on screen", () => { expect(playerBox({ left: 130, top: 260, width: 164, height: 70 }, { left: 100, top: 200, width: 800, height: 600 })) .toEqual({ left: 30, top: 60, width: 164, height: 70 }); From bd1389a793fe3bb0dce255d5c10a27e968ace0a4 Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 23:30:09 +0900 Subject: [PATCH 06/10] Make a card's preview frame 16:9 The frame was 328 by 140, about 2.34:1, a shape almost nothing a step makes: 16:9 video and pictures were cut down to a strip, and portrait and square ones came out tiny. It is now 16:9 (328 by 185 on a card with ports, 218 by 123 on one without), the commonest shape of what steps make; the file is still fitted whole, so other shapes show the backdrop around them. The height is derived from the card's width in one place, canvasNodeGeometry, instead of a 140 repeated in the card. Cards stay sized from declarations, never from the media they load. --- spec/layout.md | 3 ++- web/fx-web/src/card.ts | 4 ++-- web/fx-web/src/layout.ts | 14 +++++++++++--- web/fx-web/tests/canvas.test.ts | 8 +++++--- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/spec/layout.md b/spec/layout.md index 6fe4b8c8..33b05a9e 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -113,7 +113,8 @@ authored cells, rule 10 removes empty tracks and rule 11 sizes them. Rules 1–10 read no size, so a preview arriving never changes a cell. Footprints come from declarations: ports, settings and kind. A step whose declared output ports include an image or a -video reserves its preview area in both views, before any artifact exists. A recorded card +video reserves its preview area in both views, before any artifact exists: a 16:9 frame as wide as +the card less its padding, in which the file is fitted whole. A recorded card previews its first image output, or without one its first MP4 or WebM output, which shows its first frame until it is played (§5.7). diff --git a/web/fx-web/src/card.ts b/web/fx-web/src/card.ts index 557ccdbc..758542f7 100644 --- a/web/fx-web/src/card.ts +++ b/web/fx-web/src/card.ts @@ -95,8 +95,9 @@ export function nodeCard(node: PositionedNode, canvasId: string, index: number, } else if (!node.pending && !preview) { group.append(element("text", { x: 16, y: 64, class: "fx-canvas-port-type" }, "Port metadata not recorded")); } + const { preview_y, preview_height } = canvasNodeGeometry(node); + const frame = { x: 16, y: preview_y, width: node.width - 32, height: preview_height }; if (preview) { - const frame = { x: 16, y: canvasNodeGeometry(node).preview_y, width: node.width - 32, height: 140 }; const clipId = `${canvasId}-preview-${index}`; const clip = element("clipPath", { id: clipId }); clip.append(element("rect", { ...frame, rx: 6 })); @@ -128,7 +129,6 @@ export function nodeCard(node: PositionedNode, canvasId: string, index: number, } group.append(element("text", { x: node.width - 16, y: node.height - 22, "text-anchor": "end", class: "fx-canvas-image-count" }, count)); } else if (canvasPreviewReserved(node)) { - const frame = { x: 16, y: canvasNodeGeometry(node).preview_y, width: node.width - 32, height: 140 }; // Only a started step shimmers: the declared file may be on its way. Waiting and settled steps keep the empty frame. if (node.state === "running") { const clipId = `${canvasId}-shimmer-${index}`; diff --git a/web/fx-web/src/layout.ts b/web/fx-web/src/layout.ts index 7a294e2f..b4bcf4ca 100644 --- a/web/fx-web/src/layout.ts +++ b/web/fx-web/src/layout.ts @@ -18,13 +18,21 @@ export function canvasPreviewReserved(node: CanvasStep) { return node.preview !== undefined || (node.ports?.outputs.some((port) => port.kind === "artifact" && /^(?:image|video)(?:[/[{?]|$)/.test(port.type)) ?? false); } -/** Geometry depends on declarations, never on media loading or measured DOM text. */ +/** A preview frame's height for its width: 16:9, the commonest shape of what steps make. */ +export const PREVIEW_ASPECT = 9 / 16; + +/** + * Geometry depends on declarations, never on media loading or measured DOM text. The preview + * frame spans the card less 16 on each side. + */ export function canvasNodeGeometry(node: CanvasStep) { - if (!node.ports) return { width: 250, height: node.preview ? 246 : 106, preview_y: 61, settings_y: 0 }; + const width = node.ports ? 360 : 250; + const preview_height = Math.round((width - 32) * PREVIEW_ASPECT); + if (!node.ports) return { width, height: node.preview ? 61 + preview_height + 45 : 106, preview_y: 61, preview_height, settings_y: 0 }; const rows = Math.max(node.ports.inputs.length, node.ports.outputs.length); const settings_y = 83 + Math.max(1, rows) * PORT_ROW; const preview_y = settings_y + (node.ports.settings.length ? 29 : 5); - return { width: 360, height: preview_y + (canvasPreviewReserved(node) ? 140 + 16 : 0) + (node.kind === "workflow" ? 70 : 42), preview_y, settings_y }; + return { width, height: preview_y + (canvasPreviewReserved(node) ? preview_height + 16 : 0) + (node.kind === "workflow" ? 70 : 42), preview_y, preview_height, settings_y }; } export function canvasNodeSize(node: CanvasStep) { const { width, height } = canvasNodeGeometry(node); diff --git a/web/fx-web/tests/canvas.test.ts b/web/fx-web/tests/canvas.test.ts index 2a6e28c6..2ccd459a 100644 --- a/web/fx-web/tests/canvas.test.ts +++ b/web/fx-web/tests/canvas.test.ts @@ -16,9 +16,11 @@ function report(cells: Record, order: string[] = []): cells: Object.fromEntries(Object.entries(cells).map(([address, [column, row]]) => [address, { column, row, source: "automatic" }])) }; } -test("metadata-only nodes retain their size; image nodes reserve space before loading", () => { +test("metadata-only nodes retain their size; image nodes reserve a 16:9 frame before loading", () => { expect(canvasNodeSize(node("planned"))).toEqual({ width: 250, height: 106 }); - expect(canvasNodeSize(node("rendered", true))).toEqual({ width: 250, height: 246 }); + expect(canvasNodeSize(node("rendered", true))).toEqual({ width: 250, height: 229 }); + expect(canvasNodeGeometry(node("rendered", true)).preview_height).toBe(123); + expect(canvasNodeGeometry({ ...node("ported", true), ports: { inputs: [], outputs: [], settings: [] } }).preview_height).toBe(185); }); test("declared port rows and settings reserve space above an image preview", () => { @@ -32,7 +34,7 @@ test("declared port rows and settings reserve space above an image preview", () const last = canvasPortAnchor(placed, "input", "image_11")!; expect(last.y + 16).toBeLessThan(placed.y + geometry.settings_y); expect(geometry.settings_y + 10).toBeLessThan(geometry.preview_y); - expect(geometry.preview_y + 140).toBeLessThan(geometry.height - 22); + expect(geometry.preview_y + geometry.preview_height).toBeLessThan(geometry.height - 22); }); test("mixed image and metadata nodes remain separated with edges landing at their borders", () => { From fc70dcac4b543a6ec64311434e4ed50d67d7d0fb Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 23:40:59 +0900 Subject: [PATCH 07/10] Keep a stopped card video loaded, and show it only once it has its frame Hovering a video again went blank, its controls showed 0:00, and only then did it jump to where it had stopped: every stop unloaded the video, so every play fetched, parsed and decoded it again and seeked from the nearest keyframe. A stop now pauses and hides the player but keeps its file for 30 seconds (or until another video plays), so playing it again shows the paused frame at once and goes on from it. The card's first frame and play control also stay until the video shows its own frame there, with the player clear and passing the pointer through until then; nothing blanks while a video loads, and a video that has to seek never shows the time it is leaving. The player no longer needs the first frame as its poster. --- spec/layout.md | 5 +-- web/fx-web/src/canvas.css | 6 ++-- web/fx-web/src/video.ts | 65 +++++++++++++++++++++++++++++---------- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/spec/layout.md b/spec/layout.md index 33b05a9e..ae23a16c 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -279,8 +279,9 @@ A video plays over its card's preview, following the card as the camera and the selects nothing. Resting on it plays it muted, without taking focus or replacing a video someone played or pressed. It stops when it ends, when the pointer leaves one that is only hovered, on Esc (focus returns to its play control) and as soon as anything covers its card: another card raised -over it, its deck resting with another card in front, or the veil of a deck it is not in. Resting on -it or playing it again continues where it stopped, unless it played to its end. Its +over it, its deck resting with another card in front, or the veil of a deck it is not in. Until the +video shows its own frame, the card keeps its first frame. Resting on it or playing it again +continues where it stopped, unless it played to its end. Its controls are the viewer's own and keep their size on screen at any zoom: play, seek, time, mute and full screen, the time going first and then seek and full screen as the player gets narrow. While it plays they fade once the pointer rests. With focus in the player, Space or K plays and pauses, the diff --git a/web/fx-web/src/canvas.css b/web/fx-web/src/canvas.css index eb036cab..11ab4507 100644 --- a/web/fx-web/src/canvas.css +++ b/web/fx-web/src/canvas.css @@ -55,11 +55,13 @@ .fx-canvas-play:hover circle, .fx-canvas-play:focus-visible circle { fill: #18181b; } .fx-canvas-play:focus-visible circle { stroke: #2563eb; stroke-width: 2px; } .fx-canvas-play svg { color: #fff; pointer-events: none; } -.fx-canvas-video.is-playing .fx-canvas-play, .fx-canvas-video.is-unavailable .fx-canvas-play { display: none; } -.fx-canvas-video.is-playing .fx-canvas-node-image { visibility: hidden; } +.fx-canvas-video.is-shown .fx-canvas-play, .fx-canvas-video.is-unavailable .fx-canvas-play { display: none; } +.fx-canvas-video.is-shown .fx-canvas-node-image { visibility: hidden; } .fx-canvas-video.is-unavailable .fx-canvas-image-error { visibility: visible; } .fx-canvas-player { position: absolute; box-sizing: border-box; overflow: hidden; border-radius: 6px; outline: none; container-type: inline-size; font: 11px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; } .fx-canvas-player:focus-visible { box-shadow: 0 0 0 2px #2563eb; } +/* Until the video shows its frame the card's first frame shows through, and takes the pointer. */ +.fx-canvas-player:not(.is-shown) { opacity: 0; pointer-events: none; } .fx-canvas-player > video { display: block; width: 100%; height: 100%; object-fit: contain; } .fx-canvas-player:fullscreen { border-radius: 0; background: #000; } .fx-canvas-player.is-idle { cursor: none; } diff --git a/web/fx-web/src/video.ts b/web/fx-web/src/video.ts index f394ba88..760fdef8 100644 --- a/web/fx-web/src/video.ts +++ b/web/fx-web/src/video.ts @@ -7,6 +7,8 @@ const STILL_WIDTH = 640; const LOADS = 2; /** How long the pointer rests on a video's frame before it plays there, muted, in milliseconds. */ const HOVER_DELAY = 300; +/** How long a stopped video stays loaded, so playing it again continues at once, in milliseconds. */ +const KEEP_LOADED = 30_000; type Box = Pick; @@ -91,6 +93,10 @@ export class CardVideos { private readonly media: HTMLVideoElement; /** The video playing, and whether it plays only while hovered (muted, looping, until the pointer leaves). */ private playing: { node: string; preview: CanvasPreview; card: SVGGElement; video: SVGGElement; hover: boolean } | null = null; + /** The file the player holds, playing or kept after a stop, and whether it shows its frame yet. */ + private loaded: string | null = null; + private shown = false; + private unloadTimer: number | null = null; private frame: number | null = null; private hoverTimer: number | null = null; // Playing on hover needs a pointer that hovers, and is motion a viewer may have asked to avoid. @@ -117,6 +123,7 @@ export class CardVideos { this.failed.add(this.playing.preview.digest); this.playing.video.classList.add("is-unavailable"); this.stop(); + this.unload(); }, { signal }); this.player.addEventListener("pointerleave", () => { if (this.playing?.hover) this.stop(); }, { signal }); } @@ -169,7 +176,7 @@ export class CardVideos { const card = cards.get(this.playing.node); const video = card?.querySelector(`.fx-canvas-video[data-digest="${this.playing.preview.digest}"]`); if (!card || !video) { this.stop(); return; } - video.classList.add("is-playing"); + video.classList.toggle("is-shown", this.shown); Object.assign(this.playing, { card, video }); } @@ -185,17 +192,18 @@ export class CardVideos { else this.positions.set(playing.preview.digest, at); if (this.frame !== null) cancelAnimationFrame(this.frame); this.frame = null; - playing.video.classList.remove("is-playing"); + this.show(playing.video, false); this.controls.reset(); + // Paused, not unloaded: playing it again shows this frame at once and goes on from it. this.media.pause(); this.player.hidden = true; - this.media.removeAttribute("src"); - this.media.load(); + this.unloadTimer = window.setTimeout(() => this.unload(), KEEP_LOADED); if (focused) playing.video.querySelector(".fx-canvas-play")?.focus({ preventScroll: true }); } dispose() { this.stop(); + this.unload(); this.abort.abort(); this.observer.disconnect(); this.waiting.clear(); @@ -205,25 +213,30 @@ export class CardVideos { private play(playing: Omit, "hover">, hover = false) { this.stop(); + this.cancelUnload(); this.playing = { ...playing, hover }; - playing.video.classList.add("is-playing"); this.player.setAttribute("aria-label", playing.preview.label); this.media.muted = hover; this.media.loop = hover; this.session = new AbortController(); - // A video it stopped short of its end continues there. One starting over shows its first frame - // until the video's own does; the card's backdrop shows around it. + // A video stopped short of its end continues there: at once while the player still holds it, + // else from a seek once it loads. const { digest, url } = playing.preview; - const at = this.positions.get(digest); - const still = at === undefined ? this.ready.get(digest) : undefined; - if (still) this.media.poster = still; - else this.media.removeAttribute("poster"); - this.media.src = url; - if (at !== undefined) { - this.media.currentTime = at; - // Set before the video has loaded, the time is where it starts; a browser that drops it seeks once it can. - this.media.addEventListener("loadedmetadata", () => { if (this.media.currentTime < at - 0.5) this.media.currentTime = at; }, { once: true, signal: this.session.signal }); - } + const at = this.positions.get(digest) ?? 0; + if (this.loaded !== digest) { + this.loaded = digest; + this.media.src = url; + if (at) { + // Set before the video has loaded, the time is where it starts; a browser that drops it seeks once it can. + this.media.currentTime = at; + this.media.addEventListener("loadedmetadata", () => { if (this.media.currentTime < at - 0.5) this.media.currentTime = at; }, { once: true, signal: this.session.signal }); + } + } else if (Math.abs(this.media.currentTime - at) > 0.05) this.media.currentTime = at; + // The card keeps its first frame, and the player stays clear, until the video shows its own + // frame there: nothing blanks while it loads, and its controls never show a time it leaves. + const settle = () => { if (this.playing && this.media.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && !this.media.seeking) this.show(this.playing.video, true); }; + for (const type of ["loadeddata", "seeked", "playing"] as const) this.media.addEventListener(type, settle, { signal: this.session.signal }); + settle(); this.player.hidden = false; this.follow(); // Hovering never takes focus. @@ -234,6 +247,24 @@ export class CardVideos { if (this.hoverTimer !== null) window.clearTimeout(this.hoverTimer); this.hoverTimer = null; } + /** Whether the video shows its frame over the card, which then hides its first frame and play control. */ + private show(video: SVGGElement, shown: boolean) { + this.shown = shown; + video.classList.toggle("is-shown", shown); + this.player.classList.toggle("is-shown", shown); + } + /** Lets go of a stopped video's file and decoder. */ + private unload() { + this.cancelUnload(); + if (this.playing || this.loaded === null) return; + this.loaded = null; + this.media.removeAttribute("src"); + this.media.load(); + } + private cancelUnload() { + if (this.unloadTimer !== null) window.clearTimeout(this.unloadTimer); + this.unloadTimer = null; + } /** * Keeps the player over its card's frame every frame, as the camera and the decks move it; full From 55671fe084fd8b36440793f6c5a31e7be2968501 Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 23:46:23 +0900 Subject: [PATCH 08/10] Fade a card's video in and out over its first frame The video fades in over the card's first frame and play control, which hide once it has, and fades out on a stop as they show again at once, so switching between them no longer cuts. It is CSS transitions, with @starting-style for a kept video that shows its frame as it appears; the player now hides only after its 150 ms fade, and a play during the fade takes it back. Reduced motion keeps the cut. --- spec/layout.md | 3 ++- web/fx-web/src/canvas.css | 12 ++++++++++-- web/fx-web/src/video.ts | 9 ++++++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/spec/layout.md b/spec/layout.md index ae23a16c..0a88850c 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -280,7 +280,8 @@ selects nothing. Resting on it plays it muted, without taking focus or replacing played or pressed. It stops when it ends, when the pointer leaves one that is only hovered, on Esc (focus returns to its play control) and as soon as anything covers its card: another card raised over it, its deck resting with another card in front, or the veil of a deck it is not in. Until the -video shows its own frame, the card keeps its first frame. Resting on it or playing it again +video shows its own frame, the card keeps its first frame; the video fades in over it, and out +again when it stops. Resting on it or playing it again continues where it stopped, unless it played to its end. Its controls are the viewer's own and keep their size on screen at any zoom: play, seek, time, mute and full screen, the time going first and then seek and full screen as the player gets narrow. While it diff --git a/web/fx-web/src/canvas.css b/web/fx-web/src/canvas.css index 11ab4507..fb5aea81 100644 --- a/web/fx-web/src/canvas.css +++ b/web/fx-web/src/canvas.css @@ -55,13 +55,21 @@ .fx-canvas-play:hover circle, .fx-canvas-play:focus-visible circle { fill: #18181b; } .fx-canvas-play:focus-visible circle { stroke: #2563eb; stroke-width: 2px; } .fx-canvas-play svg { color: #fff; pointer-events: none; } -.fx-canvas-video.is-shown .fx-canvas-play, .fx-canvas-video.is-unavailable .fx-canvas-play { display: none; } -.fx-canvas-video.is-shown .fx-canvas-node-image { visibility: hidden; } +.fx-canvas-video.is-unavailable .fx-canvas-play { display: none; } +/* The video fades in over the first frame and play control, which hide once it has; they show again at once as it fades out. */ +.fx-canvas-video .fx-canvas-node-image, .fx-canvas-video .fx-canvas-play { transition: visibility 0s; } +.fx-canvas-video.is-shown .fx-canvas-node-image, .fx-canvas-video.is-shown .fx-canvas-play { visibility: hidden; transition-delay: .15s; } .fx-canvas-video.is-unavailable .fx-canvas-image-error { visibility: visible; } .fx-canvas-player { position: absolute; box-sizing: border-box; overflow: hidden; border-radius: 6px; outline: none; container-type: inline-size; font: 11px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; } .fx-canvas-player:focus-visible { box-shadow: 0 0 0 2px #2563eb; } /* Until the video shows its frame the card's first frame shows through, and takes the pointer. */ .fx-canvas-player:not(.is-shown) { opacity: 0; pointer-events: none; } +.fx-canvas-player { transition: opacity .15s ease; } +@starting-style { .fx-canvas-player.is-shown { opacity: 0; } } +@media (prefers-reduced-motion: reduce) { + .fx-canvas-player { transition: none; } + .fx-canvas-video.is-shown .fx-canvas-node-image, .fx-canvas-video.is-shown .fx-canvas-play { transition-delay: 0s; } +} .fx-canvas-player > video { display: block; width: 100%; height: 100%; object-fit: contain; } .fx-canvas-player:fullscreen { border-radius: 0; background: #000; } .fx-canvas-player.is-idle { cursor: none; } diff --git a/web/fx-web/src/video.ts b/web/fx-web/src/video.ts index 760fdef8..36712374 100644 --- a/web/fx-web/src/video.ts +++ b/web/fx-web/src/video.ts @@ -7,6 +7,8 @@ const STILL_WIDTH = 640; const LOADS = 2; /** How long the pointer rests on a video's frame before it plays there, muted, in milliseconds. */ const HOVER_DELAY = 300; +/** How long the player fades in and out over the card's first frame, in milliseconds (canvas.css). */ +const FADE = 150; /** How long a stopped video stays loaded, so playing it again continues at once, in milliseconds. */ const KEEP_LOADED = 30_000; @@ -97,6 +99,7 @@ export class CardVideos { private loaded: string | null = null; private shown = false; private unloadTimer: number | null = null; + private hideTimer: number | null = null; private frame: number | null = null; private hoverTimer: number | null = null; // Playing on hover needs a pointer that hovers, and is motion a viewer may have asked to avoid. @@ -196,13 +199,15 @@ export class CardVideos { this.controls.reset(); // Paused, not unloaded: playing it again shows this frame at once and goes on from it. this.media.pause(); - this.player.hidden = true; + // It fades out over the card's first frame, which shows again at once. + this.hideTimer = window.setTimeout(() => { this.hideTimer = null; this.player.hidden = true; }, FADE); this.unloadTimer = window.setTimeout(() => this.unload(), KEEP_LOADED); if (focused) playing.video.querySelector(".fx-canvas-play")?.focus({ preventScroll: true }); } dispose() { this.stop(); + if (this.hideTimer !== null) window.clearTimeout(this.hideTimer); this.unload(); this.abort.abort(); this.observer.disconnect(); @@ -214,6 +219,8 @@ export class CardVideos { private play(playing: Omit, "hover">, hover = false) { this.stop(); this.cancelUnload(); + if (this.hideTimer !== null) window.clearTimeout(this.hideTimer); + this.hideTimer = null; this.playing = { ...playing, hover }; this.player.setAttribute("aria-label", playing.preview.label); this.media.muted = hover; From 8794f8cdfc3297564896923b4cbe40bb774f60c5 Mon Sep 17 00:00:00 2001 From: Universe Date: Tue, 13 Oct 2026 23:49:17 +0900 Subject: [PATCH 09/10] End a canvas drag on any release, and on a move with no button held A press on a video card while its player was still clear (the hover delay, or a first frame still loading) went to the SVG; the player then appeared under the pointer and took the release, which the SVG never saw. The drag stayed open, so the next move past 4 px captured the pointer and panned the canvas with no button held, the player following its card. Firefox shows its first frame later, so it hit this more often. Releases are now heard on the canvas container, which holds the player and the toolbar, and a move with no button held ends any drag, also covering a release outside the window. --- web/fx-web/src/canvas.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/web/fx-web/src/canvas.ts b/web/fx-web/src/canvas.ts index 137cb6f2..5b638223 100644 --- a/web/fx-web/src/canvas.ts +++ b/web/fx-web/src/canvas.ts @@ -167,8 +167,9 @@ export class CanvasController { container.addEventListener("wheel", this.onWheel, { passive: false, signal: this.abort.signal }); this.svg.addEventListener("pointerdown", this.onPointerDown, { signal: this.abort.signal }); this.svg.addEventListener("pointermove", this.onPointerMove, { signal: this.abort.signal }); - this.svg.addEventListener("pointerup", this.onPointerUp, { signal: this.abort.signal }); - this.svg.addEventListener("pointercancel", this.onPointerUp, { signal: this.abort.signal }); + // A press may end off the SVG: on a video's player that appeared under it, or on the toolbar. + container.addEventListener("pointerup", this.onPointerUp, { signal: this.abort.signal }); + container.addEventListener("pointercancel", this.onPointerUp, { signal: this.abort.signal }); // The pointer moving onto the player stays on its card. this.svg.addEventListener("pointerleave", (event) => { if (!this.videos.player.contains(event.relatedTarget as Node | null)) this.hover(null); }, { signal: this.abort.signal }); // A deck card is the expand target: the capture phase runs before a card's own selection. @@ -759,6 +760,9 @@ export class CanvasController { background: !target.closest("[data-node-id], [data-connection-id], [data-deck], .fx-deck-bookmark"), veil: Boolean(target.closest(".fx-deck-veil")) }; }; private onPointerMove = (event: PointerEvent) => { + // A drag lasts only while its button is held: a release the canvas never saw (outside the + // window) ends it here rather than leaving every later move to pan. + if (this.drag && event.pointerId === this.drag.pointer && (event.buttons & 1) === 0) this.cancelDrag(); if (!this.drag) { this.hover(this.hoverAt(event.target)); return; } if (event.pointerId !== this.drag.pointer) return; if (!this.drag.moved) { From 8f18b6f4bd6373adf7755bfd9bec2fd99267236f Mon Sep 17 00:00:00 2001 From: Universe Date: Wed, 14 Oct 2026 00:07:08 +0900 Subject: [PATCH 10/10] Simplify the card player, and fix what the review found under it From a review of the whole PR for reuse, simplification, efficiency and whether each fix sits at the right depth. Fixes: - Esc on a card's player closed the viewer's details or the website's selection instead, since their own Esc handlers run first. One exported check, ownsKeys (a text field or the player keeps its own keys), now serves all three handlers. - A covered card could start its video from the keyboard (a back card's play control is still a tab stop), playing it over the front card or a deck's veil. The canvas now owns one rule for what covers a card, which the player asks before it plays and again whenever a deck rests, opens or closes, in place of two inline checks. - A press on a video's frame could outlast the hover delay, so the player appeared under it and took the release. The hover timer now waits for the release. - Clearing a deck's hover now listens for the pointer leaving the canvas, so moving onto the player needs no exception and leaving it onto a panel still restores the resting card. - Reading a first frame kept its video and canvas alive until the canvas was disposed (an abort listener never removed), and each live redraw while a frame was still being read kept the previous render's cards alive. A frame now reaches the cards that show the file when it is ready. - The player followed its card every frame by re-querying the frame and rewriting its position; it now keeps the frame and writes only changes. Simplifications: one first-frame cache instead of two; no flags that repeat the player's class or source; media listeners set up once; timers cleared directly; the play and pause glyphs built with card.ts's element() helper beside the other SVG glyphs; the canvas's own hover and motion settings passed in; one file walker for recorded runs and exports, with cardPreview walking a step's outputs itself. --- spec/layout.md | 3 +- web/fx-web/src/canvas.css | 3 +- web/fx-web/src/canvas.ts | 44 ++- web/fx-web/src/card.ts | 16 +- web/fx-web/src/export.ts | 18 +- web/fx-web/src/graph.ts | 24 +- web/fx-web/src/layout.ts | 2 +- web/fx-web/src/video-controls.ts | 27 +- web/fx-web/src/video.ts | 255 +++++++++--------- web/fx-web/tests/previews.test.ts | 7 +- web/viewer/src/main.tsx | 4 +- .../showcases/recorded-canvas-controller.ts | 5 +- 12 files changed, 210 insertions(+), 198 deletions(-) diff --git a/spec/layout.md b/spec/layout.md index 0a88850c..32b79a72 100644 --- a/spec/layout.md +++ b/spec/layout.md @@ -279,7 +279,8 @@ A video plays over its card's preview, following the card as the camera and the selects nothing. Resting on it plays it muted, without taking focus or replacing a video someone played or pressed. It stops when it ends, when the pointer leaves one that is only hovered, on Esc (focus returns to its play control) and as soon as anything covers its card: another card raised -over it, its deck resting with another card in front, or the veil of a deck it is not in. Until the +over it, its deck resting with another card in front, or the veil of a deck it is not in; a covered +card's video does not start. Until the video shows its own frame, the card keeps its first frame; the video fades in over it, and out again when it stops. Resting on it or playing it again continues where it stopped, unless it played to its end. Its diff --git a/web/fx-web/src/canvas.css b/web/fx-web/src/canvas.css index fb5aea81..66a52599 100644 --- a/web/fx-web/src/canvas.css +++ b/web/fx-web/src/canvas.css @@ -60,11 +60,10 @@ .fx-canvas-video .fx-canvas-node-image, .fx-canvas-video .fx-canvas-play { transition: visibility 0s; } .fx-canvas-video.is-shown .fx-canvas-node-image, .fx-canvas-video.is-shown .fx-canvas-play { visibility: hidden; transition-delay: .15s; } .fx-canvas-video.is-unavailable .fx-canvas-image-error { visibility: visible; } -.fx-canvas-player { position: absolute; box-sizing: border-box; overflow: hidden; border-radius: 6px; outline: none; container-type: inline-size; font: 11px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; } +.fx-canvas-player { position: absolute; box-sizing: border-box; overflow: hidden; border-radius: 6px; outline: none; container-type: inline-size; font: 11px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; transition: opacity .15s ease; } .fx-canvas-player:focus-visible { box-shadow: 0 0 0 2px #2563eb; } /* Until the video shows its frame the card's first frame shows through, and takes the pointer. */ .fx-canvas-player:not(.is-shown) { opacity: 0; pointer-events: none; } -.fx-canvas-player { transition: opacity .15s ease; } @starting-style { .fx-canvas-player.is-shown { opacity: 0; } } @media (prefers-reduced-motion: reduce) { .fx-canvas-player { transition: none; } diff --git a/web/fx-web/src/canvas.ts b/web/fx-web/src/canvas.ts index 5b638223..3acf56bb 100644 --- a/web/fx-web/src/canvas.ts +++ b/web/fx-web/src/canvas.ts @@ -10,6 +10,14 @@ import { CardVideos } from "./video"; const EXPANDED_GAP = 32; /** A card's labelled controls do what they say (§5.7): a press on one neither pans nor opens its deck. */ const CARD_CONTROLS = ".fx-canvas-open-workflow, .fx-canvas-play"; + +/** + * Whether a key event's target keeps its own keys: a text field, or a card's video player, whose + * Esc closes it. A page's own Esc (closing a panel, a deck) leaves such a target alone. + */ +export function ownsKeys(target: EventTarget | null) { + return Boolean((target as Element | null)?.closest?.("input, textarea, select, [contenteditable], .fx-canvas-player")); +} /** §5.9: expanded decks fit the viewport down to card titles (13 px) about 11 px on screen. */ const LEGIBLE_ZOOM = 11 / 13; @@ -47,6 +55,8 @@ interface Deck { badge: SVGGElement; /** The first member the closed stack shows (§5.3). */ start: number; + /** The member in front, which alone is uncovered while the deck is closed. */ + front: number; } /** A deck card under the pointer; `mark` when it is pointed at from the bookmark. */ type DeckHit = { deck: number; card: number; mark?: boolean }; @@ -162,7 +172,7 @@ export class CanvasController { const index = this.open ? this.decks.indexOf(this.open.deck) : -1; if (index >= 0) this.markFront(this.decks[index], this.frontOf(index)); }, { signal: this.abort.signal }); - this.videos = new CardVideos(container); + this.videos = new CardVideos(container, () => this.canHover.matches && !this.reducedMotion.matches, (element) => this.covered(element)); container.replaceChildren(this.svg, this.videos.player, this.toolbar, this.empty, this.deckBar, this.scrubber); container.addEventListener("wheel", this.onWheel, { passive: false, signal: this.abort.signal }); this.svg.addEventListener("pointerdown", this.onPointerDown, { signal: this.abort.signal }); @@ -171,15 +181,14 @@ export class CanvasController { container.addEventListener("pointerup", this.onPointerUp, { signal: this.abort.signal }); container.addEventListener("pointercancel", this.onPointerUp, { signal: this.abort.signal }); // The pointer moving onto the player stays on its card. - this.svg.addEventListener("pointerleave", (event) => { if (!this.videos.player.contains(event.relatedTarget as Node | null)) this.hover(null); }, { signal: this.abort.signal }); + // Leaving the canvas, not onto a video's player over its card. + container.addEventListener("pointerleave", () => this.hover(null), { signal: this.abort.signal }); // A deck card is the expand target: the capture phase runs before a card's own selection. this.svg.addEventListener("click", this.onDeckClick, { capture: true, signal: this.abort.signal }); // Raising a card moves focus out of the canvas, so Esc listens on the document while open. document.addEventListener("keydown", (event) => { if (event.key !== "Escape" || !this.open || event.defaultPrevented) return; - const target = event.target as HTMLElement | null; - // A field keeps its own keys; Esc on the card's player closes the player first. - if (target?.closest("input, textarea, select, [contenteditable], .fx-canvas-player")) return; + if (ownsKeys(event.target)) return; event.preventDefault(); this.collapse(); }, { capture: true, signal: this.abort.signal }); @@ -369,7 +378,7 @@ export class CanvasController { const names = members.map((member, at) => `${distinct ? titles[at] : member.key ?? titles[at]}${takes && member.take !== null ? ` · take ${member.take}` : ""}`); const bookmark = this.bookmark(placed, index, members, names); parent.append(bookmark); - return { placed, title, parent, cards, boxes, members, badge: group, bookmark, names, start: 0 }; + return { placed, title, parent, cards, boxes, members, badge: group, bookmark, names, start: 0, front: 0 }; } /** @@ -459,6 +468,14 @@ export class CanvasController { this.marked = card; card?.classList.add("is-marked"); } + /** + * Whether something covers this element's card: the veil of the open deck, when the card is + * outside it, or the front card of a closed deck it is in but not in front of. + */ + private covered(element: Element) { + if (this.open && !this.open.deck.cards.some((card) => card.contains(element))) return true; + return this.decks.some((deck) => deck !== this.open?.deck && deck.cards.some((card, at) => at !== deck.front && card.contains(element))); + } /** The deck card that is or holds this node's card, or -1. */ private holding(deck: Deck, id: string) { const node = this.nodes.get(id); @@ -479,9 +496,8 @@ export class CanvasController { for (const at of stack.order) if (stack.shown[at]) deck.parent.insertBefore(deck.cards[at], deck.badge); this.stack(deck, stack.slots); deck.cards.forEach((card, at) => card.classList.toggle("is-buried", !stack.shown[at])); - // Only the front card is uncovered. - const playing = this.videos.card; - if (playing && deck.cards.some((card, at) => at !== front && card.contains(playing))) this.videos.stop(); + deck.front = front; + this.videos.recheck(); this.markFront(deck, front); // One tab stop per closed deck: its front card. deck.cards.forEach((card, at) => { if (card.hasAttribute("tabindex")) card.setAttribute("tabindex", at === front ? "0" : "-1"); }); @@ -573,9 +589,6 @@ export class CanvasController { const deck = this.decks[index]; if (!deck || this.open) return; this.hover(null); - // The veil covers every card outside the deck. - const playing = this.videos.card; - if (playing && !deck.cards.some((card) => card.contains(playing))) this.videos.stop(); const sizes = deck.boxes; const card = { width: Math.max(...sizes.map((size) => size.width)), height: Math.max(...sizes.map((size) => size.height)) }; const bounds = this.container.getBoundingClientRect(); @@ -597,6 +610,7 @@ export class CanvasController { veil.style.transition = `opacity ${duration}ms ease`; veil.classList.add("is-shown"); this.open = { deck, veil, camera, grid }; + this.videos.recheck(); this.showScrubber(index); this.deckBar.replaceChildren(); const heading = document.createElement("span"); @@ -639,7 +653,11 @@ export class CanvasController { for (const card of open.deck.cards) card.style.transition = `transform ${duration}ms ease`; // They slide straight to their places in the stack; settling restacks them as it raises the front. const at = this.decks.indexOf(open.deck); - if (at >= 0) this.stack(open.deck, deckStack(open.deck.members.length, this.frontOf(at)).slots); + if (at >= 0) { + open.deck.front = this.frontOf(at); + this.stack(open.deck, deckStack(open.deck.members.length, open.deck.front).slots); + } + this.videos.recheck(); open.veil.style.transition = `opacity ${duration}ms ease`; open.veil.classList.remove("is-shown"); if (!keepCamera) this.animateCamera(() => this.viewport.restore(open.camera), duration); diff --git a/web/fx-web/src/card.ts b/web/fx-web/src/card.ts index 758542f7..e17585a9 100644 --- a/web/fx-web/src/card.ts +++ b/web/fx-web/src/card.ts @@ -2,7 +2,6 @@ import { displayPortName } from "./ports"; import { canvasNodeGeometry, canvasPreviewReserved, PORT_ROW, PORT_TOP, type PositionedNode } from "./layout"; import { stateIcon, stateIconParts } from "./state-icon"; import type { CanvasStep } from "./graph"; -import { mediaGlyph } from "./video-controls"; const SVG = "http://www.w3.org/2000/svg"; @@ -41,6 +40,17 @@ export function cardStatus(node: Pick = {}) { + const svg = element("svg", { viewBox: "0 0 24 24", fill: "currentColor", "aria-hidden": "true", ...attributes }); + if (kind === "play") svg.append(element("path", { d: "M7 4.5v15a1 1 0 0 0 1.53.85l12-7.5a1 1 0 0 0 0-1.7l-12-7.5A1 1 0 0 0 7 4.5z" })); + else for (const x of [6, 14]) svg.append(element("rect", { x, y: 5, width: 4, height: 14, rx: 1.25 })); + return svg; +} + /** A state's glyph as a nested SVG square of `size` at (x, y). */ export function stateIconElement(state: string, x: number, y: number, size: number) { const icon = stateIcon(state); @@ -59,6 +69,7 @@ export function short(text: string, limit: number) { return [...text].length > l */ export function nodeCard(node: PositionedNode, canvasId: string, index: number, signal: AbortSignal, onOpen: (scope: string) => void) { const preview = node.preview; + const { settings_y, preview_y, preview_height } = canvasNodeGeometry(node); const portsLabel = node.ports ? `, inputs: ${node.ports.inputs.map((port) => `${displayPortName(port.name)} (${port.type}, ${port.kind})`).join(", ") || "none"}, outputs: ${node.ports.outputs.map((port) => `${displayPortName(port.name)} (${port.type}, ${port.kind})`).join(", ") || "none"}` : node.pending ? "" : ", port metadata not recorded"; const scopeLabel = node.kind === "workflow" ? `, imported workflow, ${node.child_count ?? 0} steps, ${node.failure_count ?? 0} failed` : ""; const state = node.state === "waiting" && node.waiting_for ? `waiting for ${node.waiting_for}` : node.state; @@ -89,13 +100,12 @@ export function nodeCard(node: PositionedNode, canvasId: string, index: number, if (!ports.length) group.append(element("text", { x: textX, y: PORT_TOP + 3, "text-anchor": direction === "input" ? "start" : "end", class: "fx-canvas-port-type" }, "None")); } if (node.ports.settings.length) { - const settings = element("text", { x: 16, y: canvasNodeGeometry(node).settings_y + 10, class: "fx-canvas-settings" }, short(`Settings · ${node.ports.settings.map((setting) => setting.name).join(", ")}`, 52)); + const settings = element("text", { x: 16, y: settings_y + 10, class: "fx-canvas-settings" }, short(`Settings · ${node.ports.settings.map((setting) => setting.name).join(", ")}`, 52)); group.append(settings); } } else if (!node.pending && !preview) { group.append(element("text", { x: 16, y: 64, class: "fx-canvas-port-type" }, "Port metadata not recorded")); } - const { preview_y, preview_height } = canvasNodeGeometry(node); const frame = { x: 16, y: preview_y, width: node.width - 32, height: preview_height }; if (preview) { const clipId = `${canvasId}-preview-${index}`; diff --git a/web/fx-web/src/export.ts b/web/fx-web/src/export.ts index 96b486c5..d2b38a66 100644 --- a/web/fx-web/src/export.ts +++ b/web/fx-web/src/export.ts @@ -1,5 +1,5 @@ import { parseViewerRun, type ViewerRun } from "./index"; -import { cardPreview, flatCanvasGraph, parseLayoutReport, type CanvasGraph, type LayoutReport } from "./graph"; +import { cardPreview, flatCanvasGraph, parseLayoutReport, recordedFileDigests, type CanvasGraph, type LayoutReport } from "./graph"; import { projectScopes } from "./scopes"; export interface ExportArtifact { @@ -91,14 +91,6 @@ function pendingRepeat(item: unknown) { && (item.waiting_on === undefined || (Array.isArray(item.waiting_on) && item.waiting_on.every((id) => typeof id === "string") && new Set(item.waiting_on).size === item.waiting_on.length)); } -/** Only FX value variants contain artifact references; opaque JSON is never interpreted as wiring. */ -export function exportFileDigests(value: unknown): string[] { - if (!record(value)) return []; - if (record(value.file)) return digest(value.file.digest) ? [value.file.digest] : []; - if (Array.isArray(value.list)) return value.list.flatMap(exportFileDigests); - if (Array.isArray(value.collection)) return value.collection.flatMap((entry) => Array.isArray(entry) && entry.length === 2 ? exportFileDigests(entry[1]) : []); - return []; -} function validateReferences(view: ViewerRun, layout: LayoutReport, artifacts: ExportArtifact[]) { const ids = new Set(view.nodes.map((node) => node.id)); @@ -134,7 +126,7 @@ function validateReferences(view: ViewerRun, layout: LayoutReport, artifacts: Ex } const inventory = new Set(artifacts.map((item) => item.digest)); for (const values of [view.inputs, view.outputs, ...view.nodes.flatMap((node) => [node.with, node.outputs])]) { - for (const value of Object.values(values)) if (exportFileDigests(value).some((id) => !inventory.has(id))) refuse("an artifact reference is missing from the inventory."); + for (const value of Object.values(values)) if (recordedFileDigests(value).some((id) => !inventory.has(id))) refuse("an artifact reference is missing from the inventory."); } const addresses = new Set([ ...view.nodes.map((node) => node.step ?? node.path), @@ -230,11 +222,11 @@ export function exportCanvasGraph(loaded: LoadedRunExport, scope: string | null for (const node of flat.nodes) { const source = view.nodes.find((item) => item.id === node.source_id); if (!source) continue; - const preview = cardPreview(Object.entries(source.outputs).flatMap(([port, value]) => exportFileDigests(value).flatMap((id) => { + const preview = cardPreview(source.outputs, (id) => { const artifact = inventory.get(id); const url = artifact ? exportArtifactUrl(loaded, id) : null; - return artifact && url ? [{ digest: id, media_type: artifact.media_type, url, label: `${port}: ${artifact.name}` }] : []; - }))); + return artifact && url ? { media_type: artifact.media_type, name: artifact.name, url } : null; + }); if (preview) node.preview = preview; } return projectScopes(view, flat, scope); diff --git a/web/fx-web/src/graph.ts b/web/fx-web/src/graph.ts index 997e2109..00381f9e 100644 --- a/web/fx-web/src/graph.ts +++ b/web/fx-web/src/graph.ts @@ -199,8 +199,8 @@ const compareText = (left: string, right: string) => left < right ? -1 : left > const compareNamed = (left: { name: string; kind?: string }, right: { name: string; kind?: string }) => compareText(left.name, right.name) || compareText(left.kind ?? "", right.kind ?? ""); -/** Follow recorded FX value variants; ordinary JSON values remain opaque. */ -function recordedFileDigests(value: unknown): string[] { +/** Follow recorded FX value variants; ordinary JSON values remain opaque and are never read as wiring. */ +export function recordedFileDigests(value: unknown): string[] { if (!isRecord(value)) return []; if (isRecord(value.file)) return isDigest(value.file.digest) ? [value.file.digest as string] : []; if (Array.isArray(value.list)) return value.list.flatMap(recordedFileDigests); @@ -210,14 +210,15 @@ function recordedFileDigests(value: unknown): string[] { } /** - * A card's preview from its output files in output order: the first image, or without one the - * first video. Callers pass only files they resolved to a verified URL. + * A card's preview from a step's output files, in output order: the first image, or without one + * the first video. `resolve` gives a file's type, name and verified URL, or null to leave it out. */ -export function cardPreview(files: Iterable<{ digest: string; media_type: string; url: string; label: string }>): CanvasPreview | undefined { +export function cardPreview(outputs: Record, resolve: (digest: string) => { media_type: string; name: string; url: string } | null): CanvasPreview | undefined { const found = { image: new Map(), video: new Map() }; - for (const { digest, media_type, url, label } of files) { - const kind = previewKinds[media_type]; - if (kind && !found[kind].has(digest)) found[kind].set(digest, { url, label }); + for (const [port, value] of Object.entries(outputs)) for (const digest of recordedFileDigests(value)) { + const file = resolve(digest); + const kind = file && previewKinds[file.media_type]; + if (file && kind && !found[kind].has(digest)) found[kind].set(digest, { url: file.url, label: `${port}: ${file.name}` }); } const kind = found.image.size ? "image" : found.video.size ? "video" : undefined; const first = kind && found[kind].entries().next().value; @@ -226,11 +227,10 @@ export function cardPreview(files: Iterable<{ digest: string; media_type: string /** Select a recorded output picture or video from the host's verified artifact inventory. */ function recordedPreview(node: ViewerRun["nodes"][number], artifacts: Map, apiBase: string): CanvasPreview | undefined { - return cardPreview(Object.entries(node.outputs).flatMap(([port, value]) => recordedFileDigests(value).flatMap((digest) => { + return cardPreview(node.outputs, (digest) => { const artifact = artifacts.get(digest); - return artifact?.available && artifact.url === `${apiBase}/artifacts/${digest}` - ? [{ digest, media_type: artifact.kind, url: artifact.url, label: `${port}: ${artifact.name}` }] : []; - }))); + return artifact?.available && artifact.url === `${apiBase}/artifacts/${digest}` ? { media_type: artifact.kind, name: artifact.name, url: artifact.url } : null; + }); } /** Project declared wiring for display; no expressions or execution decisions are evaluated. */ diff --git a/web/fx-web/src/layout.ts b/web/fx-web/src/layout.ts index b4bcf4ca..da85a107 100644 --- a/web/fx-web/src/layout.ts +++ b/web/fx-web/src/layout.ts @@ -19,7 +19,7 @@ export function canvasPreviewReserved(node: CanvasStep) { } /** A preview frame's height for its width: 16:9, the commonest shape of what steps make. */ -export const PREVIEW_ASPECT = 9 / 16; +const PREVIEW_ASPECT = 9 / 16; /** * Geometry depends on declarations, never on media loading or measured DOM text. The preview diff --git a/web/fx-web/src/video-controls.ts b/web/fx-web/src/video-controls.ts index e837639c..d44c9587 100644 --- a/web/fx-web/src/video-controls.ts +++ b/web/fx-web/src/video-controls.ts @@ -1,29 +1,11 @@ import { createElement, Maximize, Minimize, Volume2, VolumeX, type IconNode } from "lucide"; +import { mediaGlyph } from "./card"; /** How long the pointer rests before a playing video's controls fade, in milliseconds. */ const IDLE_DELAY = 2000; /** How far the arrow keys seek, in seconds. */ const SEEK_STEP = 5; -/** - * Filled play and pause glyphs, on a 24-unit square, drawn in the current colour. Unlike the rest - * of the viewer's chrome they are not Lucide's: its outlined ones read poorly on a video. - */ -export function mediaGlyph(kind: "play" | "pause", attributes: Record = {}) { - const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); - for (const [name, value] of Object.entries({ viewBox: "0 0 24 24", fill: "currentColor", "aria-hidden": "true", ...attributes })) svg.setAttribute(name, String(value)); - if (kind === "play") { - const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); - path.setAttribute("d", "M7 4.5v15a1 1 0 0 0 1.53.85l12-7.5a1 1 0 0 0 0-1.7l-12-7.5A1 1 0 0 0 7 4.5z"); - svg.append(path); - } else for (const x of [6, 14]) { - const bar = document.createElementNS("http://www.w3.org/2000/svg", "rect"); - for (const [name, value] of Object.entries({ x, y: 5, width: 4, height: 14, rx: 1.25 })) bar.setAttribute(name, String(value)); - svg.append(bar); - } - return svg; -} - /** A video time as players show it: "0:07", "12:30", "1:02:03". */ export function videoTime(seconds: number) { const whole = Number.isFinite(seconds) ? Math.max(0, Math.floor(seconds)) : 0; @@ -107,7 +89,7 @@ export class VideoControls { element.addEventListener("pointerdown", (event) => { taken = press(event.target === video); this.wake(); }, { signal }); video.addEventListener("click", () => { if (taken) taken = false; else this.toggle(); }, { signal }); element.addEventListener("pointermove", () => this.wake(), { signal }); - element.addEventListener("pointerleave", () => { if (!video.paused) this.rest(); }, { signal }); + element.addEventListener("pointerleave", () => this.rest(), { signal }); element.addEventListener("focusin", () => this.wake(), { signal }); element.addEventListener("keydown", (event) => { const target = event.target === seek ? "slider" : (event.target as Element).closest("button") ? "button" : "other"; @@ -177,8 +159,9 @@ export class VideoControls { const buffered = video.buffered.length ? video.buffered.end(video.buffered.length - 1) : 0; seek.style.setProperty("--fx-played", share(video.currentTime)); seek.style.setProperty("--fx-buffered", share(buffered)); - const text = `${videoTime(video.currentTime)} / ${videoTime(duration)}`; - seek.setAttribute("aria-valuetext", `${videoTime(video.currentTime)} of ${videoTime(duration)}`); + const [at, length] = [videoTime(video.currentTime), videoTime(duration)]; + const text = `${at} / ${length}`; + seek.setAttribute("aria-valuetext", `${at} of ${length}`); if (this.time.textContent !== text) this.time.textContent = text; } diff --git a/web/fx-web/src/video.ts b/web/fx-web/src/video.ts index 36712374..208ddbca 100644 --- a/web/fx-web/src/video.ts +++ b/web/fx-web/src/video.ts @@ -12,34 +12,32 @@ const FADE = 150; /** How long a stopped video stays loaded, so playing it again continues at once, in milliseconds. */ const KEEP_LOADED = 30_000; -type Box = Pick; - /** Where a video stopped at `time` plays from next: there, or from the start once it has played through. */ export function resumeAt(time: number, duration: number): number | null { if (!Number.isFinite(time) || time <= 0) return null; return Number.isFinite(duration) && time >= duration - 0.25 ? null : time; } -/** Where the player goes, in the host's pixels: over the frame as it is on screen now. */ -export function playerBox(frame: Box, host: Box) { - return { left: frame.left - host.left, top: frame.top - host.top, width: frame.width, height: frame.height }; -} - /** - * A video's first frame as a picture, or null when the video does not load or this page may not - * read its pixels: a file on another host must allow this page (CORS), or it does not load. + * A video's first frame drawn on a canvas, or null when the video does not load or this page may + * not read its pixels: a file on another host must allow this page (CORS), or it does not load. + * The video lets go of its file before the caller encodes the frame. */ -async function firstFrame(url: string, signal: AbortSignal): Promise { +async function drawFirstFrame(url: string, signal: AbortSignal): Promise { + if (signal.aborted) return null; const video = document.createElement("video"); video.crossOrigin = "anonymous"; video.muted = true; video.playsInline = true; video.preload = "auto"; + let settle = (_loaded: boolean) => {}; + const abort = () => settle(false); + signal.addEventListener("abort", abort, { once: true }); try { const loaded = await new Promise((resolve) => { + settle = resolve; video.addEventListener("loadeddata", () => resolve(true), { once: true }); video.addEventListener("error", () => resolve(false), { once: true }); - signal.addEventListener("abort", () => resolve(false), { once: true }); video.src = url; }); if (!loaded || !video.videoWidth || !video.videoHeight) return null; @@ -48,65 +46,77 @@ async function firstFrame(url: string, signal: AbortSignal): Promise((resolve) => canvas.toBlob(resolve, "image/webp")); + return canvas; } catch { return null; } finally { + signal.removeEventListener("abort", abort); video.removeAttribute("src"); video.load(); } } +/** A video's first frame as a picture, or null (drawFirstFrame). */ +async function firstFrame(url: string, signal: AbortSignal): Promise { + const canvas = await drawFirstFrame(url, signal); + if (!canvas) return null; + try { + // WebP keeps transparency; a browser that cannot write it gives PNG. + return await new Promise((resolve) => canvas.toBlob(resolve, "image/webp")); + } catch { + return null; + } finally { + canvas.width = canvas.height = 0; + } +} + +/** The playing video: its step, its file, its card's video group and frame, and whether it only plays while hovered. */ +interface Playing { node: string; preview: CanvasPreview; video: SVGGElement; frame: Element; hover: boolean } + /** * Video previews on node cards. SVG has no video that every browser draws in place: Safari draws * an HTML video inside `foreignObject` outside the SVG's transforms and above the rest of it. So a * card shows its video's first frame as an ordinary SVG image, read once per file when the card * first comes into view, and one HTML player with the viewer's own controls (video-controls.ts), * laid over the card's frame, plays it: with sound from its play control, or muted and looping - * while the pointer rests on the frame. The canvas stops the player when anything covers its card. + * while the pointer rests on the frame. It never plays, or goes on playing, over a covered card: + * the canvas says what covers one. */ export class CardVideos { - /** First frames by digest: an object URL, or null when the file could not be read. */ - private readonly stills = new Map>(); - private readonly ready = new Map(); + /** First frames by digest: an object URL, null when the file could not be read, undefined while reading. */ + private readonly stills = new Map(); /** Files the player could not play: their cards say so and offer no play control. */ private readonly failed = new Set(); /** Where each file stopped short of its end: hovering or playing it again continues there. */ private readonly positions = new Map(); - /** What only the current video listens for; a stop ends it. */ - private session = new AbortController(); - private readonly waiting = new Map void>(); + /** First frames waiting for their card to come into view. */ + private readonly waiting = new Map(); private readonly observer = new IntersectionObserver((entries) => { for (const entry of entries) { - const load = entry.isIntersecting ? this.waiting.get(entry.target) : undefined; - if (!load) continue; + const preview = entry.isIntersecting ? this.waiting.get(entry.target) : undefined; + if (!preview) continue; this.observer.unobserve(entry.target); this.waiting.delete(entry.target); - load(); + this.read(preview); } }, { rootMargin: "200px" }); - private readonly queue: (() => void)[] = []; + private readonly queue: { digest: string; url: string }[] = []; private loading = 0; private readonly abort = new AbortController(); private readonly controls: VideoControls; /** The player: the video and its controls, laid over the playing card's frame. */ readonly player: HTMLDivElement; private readonly media: HTMLVideoElement; - /** The video playing, and whether it plays only while hovered (muted, looping, until the pointer leaves). */ - private playing: { node: string; preview: CanvasPreview; card: SVGGElement; video: SVGGElement; hover: boolean } | null = null; - /** The file the player holds, playing or kept after a stop, and whether it shows its frame yet. */ - private loaded: string | null = null; - private shown = false; - private unloadTimer: number | null = null; - private hideTimer: number | null = null; - private frame: number | null = null; - private hoverTimer: number | null = null; - // Playing on hover needs a pointer that hovers, and is motion a viewer may have asked to avoid. - private readonly canHover = window.matchMedia("(hover: hover)"); - private readonly reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)"); + private playing: Playing | null = null; + /** The player's last placement, written only when it changes. */ + private box = ""; + // Timer and frame handles; clearing 0 or a spent one does nothing. + private following = 0; + private hoverTimer = 0; + private hideTimer = 0; + private unloadTimer = 0; - constructor(private readonly host: HTMLElement) { + constructor(private readonly host: HTMLElement, private readonly hoverPlays: () => boolean, private readonly covered: (element: Element) => boolean) { const signal = this.abort.signal; // Leaving a hovered video stops it. Pressing it keeps it playing, and a press on its picture // turns its sound on rather than pausing it. @@ -128,23 +138,28 @@ export class CardVideos { this.stop(); this.unload(); }, { signal }); + for (const type of ["loadeddata", "seeked", "playing"] as const) this.media.addEventListener(type, this.settle, { signal }); + // A time set before the video loaded is where it starts; a browser that drops it seeks once it can. + this.media.addEventListener("loadedmetadata", () => { + const at = this.playing && this.positions.get(this.playing.preview.digest); + if (at && this.media.currentTime < at - 0.5) this.media.currentTime = at; + }, { signal }); this.player.addEventListener("pointerleave", () => { if (this.playing?.hover) this.stop(); }, { signal }); } - /** The card whose video plays. */ - get card(): SVGGElement | null { return this.playing?.card ?? null; } - /** Wires a drawn card's video: its first frame once the card is in view, and its play control. */ attach(card: SVGGElement, node: string, preview: CanvasPreview, signal: AbortSignal) { const video = card.querySelector(".fx-canvas-video"); const image = video?.querySelector("image"); + const frame = video?.querySelector(".fx-canvas-image-border"); const play = video?.querySelector(".fx-canvas-play"); - if (!video || !image || !play) return; + if (!video || !image || !frame || !play) return; if (this.failed.has(preview.digest)) video.classList.add("is-unavailable"); - const show = (url: string | null) => { if (url && !signal.aborted) image.setAttribute("href", url); }; - if (this.ready.has(preview.digest)) show(this.ready.get(preview.digest)!); - else { - this.waiting.set(image, () => void this.still(preview).then(show)); + const still = this.stills.get(preview.digest); + if (still) image.setAttribute("href", still); + // A first frame already being read reaches this card when it is ready (read). + else if (!this.stills.has(preview.digest)) { + this.waiting.set(image, preview); this.observer.observe(image); signal.addEventListener("abort", () => { this.observer.unobserve(image); this.waiting.delete(image); }, { once: true }); } @@ -152,23 +167,26 @@ export class CardVideos { const start = (event: Event) => { event.preventDefault(); event.stopPropagation(); - this.play({ node, preview, card, video }); + this.play({ node, preview, video, frame }); }; play.addEventListener("click", start, { signal }); play.addEventListener("keydown", (event) => { if (event.key === "Enter" || event.key === " ") start(event); }, { signal }); - // Resting on the frame plays it muted, unless a video played with sound is on. - video.addEventListener("pointerenter", () => { - this.cancelHover(); - if (!this.canHover.matches || this.reducedMotion.matches || this.failed.has(preview.digest) || (this.playing && !this.playing.hover)) return; + // Resting on the frame plays it muted, unless a video played with sound is on. A press waits + // for its release, so the player never appears under a click. + const rest = () => { + window.clearTimeout(this.hoverTimer); + if (!this.hoverPlays() || this.failed.has(preview.digest)) return; this.hoverTimer = window.setTimeout(() => { - this.hoverTimer = null; - if (video.isConnected) this.play({ node, preview, card, video }, true); + if (video.isConnected && !(this.playing && (!this.playing.hover || this.playing.video === video))) this.play({ node, preview, video, frame }, true); }, HOVER_DELAY); - }, { signal }); + }; + video.addEventListener("pointerenter", rest, { signal }); + video.addEventListener("pointerdown", () => window.clearTimeout(this.hoverTimer), { signal }); + video.addEventListener("pointerup", rest, { signal }); // The player appears under the pointer: moving onto it stays on the frame. video.addEventListener("pointerleave", (event) => { if (this.player.contains(event.relatedTarget as Node | null)) return; - this.cancelHover(); + window.clearTimeout(this.hoverTimer); if (this.playing?.hover && this.playing.video === video) this.stop(); }, { signal }); } @@ -176,138 +194,133 @@ export class CardVideos { /** After a redraw the player stays with its step while the step's card shows the same file. */ redrawn(cards: ReadonlyMap) { if (!this.playing) return; - const card = cards.get(this.playing.node); - const video = card?.querySelector(`.fx-canvas-video[data-digest="${this.playing.preview.digest}"]`); - if (!card || !video) { this.stop(); return; } - video.classList.toggle("is-shown", this.shown); - Object.assign(this.playing, { card, video }); + const video = cards.get(this.playing.node)?.querySelector(`.fx-canvas-video[data-digest="${this.playing.preview.digest}"]`); + const frame = video?.querySelector(".fx-canvas-image-border"); + if (!video || !frame) { this.stop(); return; } + video.classList.toggle("is-shown", this.player.classList.contains("is-shown")); + Object.assign(this.playing, { video, frame }); + } + + /** Stops the video if something now covers its card. */ + recheck() { + if (this.playing && this.covered(this.playing.video)) this.stop(); } stop() { - this.cancelHover(); + window.clearTimeout(this.hoverTimer); const playing = this.playing; if (!playing) return; const focused = this.player.contains(document.activeElement); this.playing = null; - this.session.abort(); const at = resumeAt(this.media.currentTime, this.media.duration); if (at === null) this.positions.delete(playing.preview.digest); else this.positions.set(playing.preview.digest, at); - if (this.frame !== null) cancelAnimationFrame(this.frame); - this.frame = null; + cancelAnimationFrame(this.following); + this.box = ""; this.show(playing.video, false); this.controls.reset(); - // Paused, not unloaded: playing it again shows this frame at once and goes on from it. + // Paused, not unloaded: playing it again shows this frame at once and goes on from it. It fades + // out over the card's first frame, which shows again at once. this.media.pause(); - // It fades out over the card's first frame, which shows again at once. - this.hideTimer = window.setTimeout(() => { this.hideTimer = null; this.player.hidden = true; }, FADE); + this.hideTimer = window.setTimeout(() => { this.player.hidden = true; }, FADE); this.unloadTimer = window.setTimeout(() => this.unload(), KEEP_LOADED); if (focused) playing.video.querySelector(".fx-canvas-play")?.focus({ preventScroll: true }); } dispose() { this.stop(); - if (this.hideTimer !== null) window.clearTimeout(this.hideTimer); + window.clearTimeout(this.hideTimer); this.unload(); this.abort.abort(); this.observer.disconnect(); this.waiting.clear(); - for (const url of this.ready.values()) if (url) URL.revokeObjectURL(url); - this.ready.clear(); + for (const still of this.stills.values()) if (still) URL.revokeObjectURL(still); + this.stills.clear(); } - private play(playing: Omit, "hover">, hover = false) { + private play(playing: Omit, hover = false) { + if (this.covered(playing.video)) return; this.stop(); - this.cancelUnload(); - if (this.hideTimer !== null) window.clearTimeout(this.hideTimer); - this.hideTimer = null; + window.clearTimeout(this.hideTimer); + window.clearTimeout(this.unloadTimer); this.playing = { ...playing, hover }; this.player.setAttribute("aria-label", playing.preview.label); this.media.muted = hover; this.media.loop = hover; - this.session = new AbortController(); // A video stopped short of its end continues there: at once while the player still holds it, - // else from a seek once it loads. + // else from where it starts loading. const { digest, url } = playing.preview; const at = this.positions.get(digest) ?? 0; - if (this.loaded !== digest) { - this.loaded = digest; + if (this.media.getAttribute("src") !== url) { this.media.src = url; - if (at) { - // Set before the video has loaded, the time is where it starts; a browser that drops it seeks once it can. - this.media.currentTime = at; - this.media.addEventListener("loadedmetadata", () => { if (this.media.currentTime < at - 0.5) this.media.currentTime = at; }, { once: true, signal: this.session.signal }); - } + if (at) this.media.currentTime = at; } else if (Math.abs(this.media.currentTime - at) > 0.05) this.media.currentTime = at; - // The card keeps its first frame, and the player stays clear, until the video shows its own - // frame there: nothing blanks while it loads, and its controls never show a time it leaves. - const settle = () => { if (this.playing && this.media.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && !this.media.seeking) this.show(this.playing.video, true); }; - for (const type of ["loadeddata", "seeked", "playing"] as const) this.media.addEventListener(type, settle, { signal: this.session.signal }); - settle(); + this.settle(); this.player.hidden = false; this.follow(); // Hovering never takes focus. if (!hover) this.player.focus({ preventScroll: true }); void this.media.play().catch(() => {}); } - private cancelHover() { - if (this.hoverTimer !== null) window.clearTimeout(this.hoverTimer); - this.hoverTimer = null; - } + + /** + * The card keeps its first frame, and the player stays clear, until the video shows its own frame + * there: nothing blanks while it loads, and its controls never show a time it leaves. + */ + private settle = () => { + if (this.playing && this.media.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && !this.media.seeking) this.show(this.playing.video, true); + }; /** Whether the video shows its frame over the card, which then hides its first frame and play control. */ private show(video: SVGGElement, shown: boolean) { - this.shown = shown; video.classList.toggle("is-shown", shown); this.player.classList.toggle("is-shown", shown); } /** Lets go of a stopped video's file and decoder. */ private unload() { - this.cancelUnload(); - if (this.playing || this.loaded === null) return; - this.loaded = null; + window.clearTimeout(this.unloadTimer); + if (this.playing || !this.media.hasAttribute("src")) return; this.media.removeAttribute("src"); this.media.load(); } - private cancelUnload() { - if (this.unloadTimer !== null) window.clearTimeout(this.unloadTimer); - this.unloadTimer = null; - } /** * Keeps the player over its card's frame every frame, as the camera and the decks move it; full * screen, the browser places it. */ private follow = () => { - const frame = this.playing?.video.querySelector(".fx-canvas-image-border"); - if (!frame) return; + if (!this.playing) return; if (!this.controls.fullscreened) { - const box = playerBox(frame.getBoundingClientRect(), this.host.getBoundingClientRect()); - for (const side of ["left", "top", "width", "height"] as const) this.player.style[side] = `${box[side]}px`; + const frame = this.playing.frame.getBoundingClientRect(), host = this.host.getBoundingClientRect(); + const box = [frame.left - host.left, frame.top - host.top, frame.width, frame.height]; + const key = box.join(); + if (key !== this.box) { + this.box = key; + (["left", "top", "width", "height"] as const).forEach((side, at) => { this.player.style[side] = `${box[at]}px`; }); + } } - this.frame = requestAnimationFrame(this.follow); + this.following = requestAnimationFrame(this.follow); }; - /** A file's first frame, read once and at most `LOADS` at a time. */ - private still(preview: CanvasPreview): Promise { - let still = this.stills.get(preview.digest); - if (still) return still; - const signal = this.abort.signal; - still = new Promise((resolve) => { this.queue.push(resolve); this.next(); }) - .then(() => firstFrame(preview.url, signal)) - .then((blob) => { - this.loading--; - this.next(); - const url = blob && !signal.aborted ? URL.createObjectURL(blob) : null; - this.ready.set(preview.digest, url); - return url; - }); - this.stills.set(preview.digest, still); - return still; + /** Asks for a file's first frame once; at most `LOADS` read at a time. */ + private read({ digest, url }: CanvasPreview) { + if (this.stills.has(digest)) return; + this.stills.set(digest, undefined); + this.queue.push({ digest, url }); + this.next(); } private next() { while (this.loading < LOADS && this.queue.length) { + const { digest, url } = this.queue.shift()!; this.loading++; - this.queue.shift()!(); + void firstFrame(url, this.abort.signal).then((blob) => { + this.loading--; + this.next(); + if (this.abort.signal.aborted) return; + const still = blob ? URL.createObjectURL(blob) : null; + this.stills.set(digest, still); + // Whichever cards show the file now; a card drawn since the read began among them. + if (still) for (const image of this.host.querySelectorAll(`.fx-canvas-video[data-digest="${digest}"] image`)) image.setAttribute("href", still); + }); } } } diff --git a/web/fx-web/tests/previews.test.ts b/web/fx-web/tests/previews.test.ts index 86314264..8157a059 100644 --- a/web/fx-web/tests/previews.test.ts +++ b/web/fx-web/tests/previews.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { canvasGraph, type Artifact, type ViewerRun } from "../src/index"; -import { playerBox, resumeAt } from "../src/video"; +import { resumeAt } from "../src/video"; import { graph } from "./graph.test"; const sourceDigest = "a".repeat(64); @@ -126,9 +126,4 @@ describe("recorded canvas previews", () => { // A stream of unknown length always continues. expect(resumeAt(40, Infinity)).toBe(40); }); - - test("the player covers its card's frame as it is on screen", () => { - expect(playerBox({ left: 130, top: 260, width: 164, height: 70 }, { left: 100, top: 200, width: 800, height: 600 })) - .toEqual({ left: 30, top: 60, width: 164, height: 70 }); - }); }); diff --git a/web/viewer/src/main.tsx b/web/viewer/src/main.tsx index 50392260..77f97572 100644 --- a/web/viewer/src/main.tsx +++ b/web/viewer/src/main.tsx @@ -1,7 +1,7 @@ import { StrictMode, useEffect, useRef, useState, useSyncExternalStore, type ReactNode } from "react"; import { createRoot } from "react-dom/client"; import { ArrowLeft, ArrowUpRight, Check, ChevronRight, Copy, FileText, List, RefreshCw, Square, Workflow, X } from "lucide-react"; -import { ServiceIndexProjection, ViewerController, elapsedLabel, ViewerEntryController, type Artifact, type CanvasStep, type GraphDocument, type GraphInstance, type PendingRepeat, type ServiceEntry, type ServiceIndex, type ViewerRun } from "@grida/fx-web"; +import { ServiceIndexProjection, ViewerController, elapsedLabel, ownsKeys, ViewerEntryController, type Artifact, type CanvasStep, type GraphDocument, type GraphInstance, type PendingRepeat, type ServiceEntry, type ServiceIndex, type ViewerRun } from "@grida/fx-web"; import { StateIcon, StateLabel, Status, Values, WorkflowCanvas, money, type WorkflowCanvasHandle } from "@grida/fx-react"; import "./style.css"; @@ -179,7 +179,7 @@ function WorkflowView() { useEffect(() => { if (!open && !steps) return; const close = (event: KeyboardEvent) => { - if (event.key !== "Escape" || event.defaultPrevented || (event.target as Element | null)?.closest?.("input, textarea, select, [contenteditable]")) return; + if (event.key !== "Escape" || event.defaultPrevented || ownsKeys(event.target)) return; event.preventDefault(); if (open) { setAbout(false); controller.select(null); } else setSteps(false); }; diff --git a/www/src/components/showcases/recorded-canvas-controller.ts b/www/src/components/showcases/recorded-canvas-controller.ts index a3b8e6f4..4eaa56f9 100644 --- a/www/src/components/showcases/recorded-canvas-controller.ts +++ b/www/src/components/showcases/recorded-canvas-controller.ts @@ -1,4 +1,4 @@ -import { CanvasController, ExportViewerController, exportArtifactUrl, type CanvasGraph, type ExportViewerState, type LoadedRunExport, type PinnedRunExport } from "@grida/fx-web"; +import { CanvasController, ExportViewerController, exportArtifactUrl, ownsKeys, type CanvasGraph, type ExportViewerState, type LoadedRunExport, type PinnedRunExport } from "@grida/fx-web"; export interface Recording extends PinnedRunExport { id: string; label: string } export interface CanvasFragment { graph: boolean; recording: string | null; scope: string | null; node: string | null } @@ -118,7 +118,8 @@ export class RecordedCanvasController { } }; private keydown = (event: KeyboardEvent) => { - if (event.key === "Escape" && this.viewer.getSnapshot().selected !== null && !event.defaultPrevented) { + // A text field or a card's video player keeps its own Esc. + if (event.key === "Escape" && this.viewer.getSnapshot().selected !== null && !event.defaultPrevented && !ownsKeys(event.target)) { event.preventDefault(); this.viewer.select(null); this.writeFragment(); } if (!(event.target instanceof HTMLElement) || !event.target.matches("[data-mode]")) return;