From f8439618923edc872ca327188b1fd0a62aedc22a Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Tue, 6 Oct 2026 14:17:25 -0400 Subject: [PATCH] Lay out reply media as galleries with a shared viewer (0.261.260) Runs of images and clips in a V2 reply render as tiles, three to a row, with captions; evidence lists put each item's clip and still in a row. A viewer steps through a gallery (buttons and arrow keys), plays clips, shows images at actual size and downloads. The drawer groups Images, Videos and Audio, plays and downloads recordings, and scrolls to an item's message. Escape in a drawer dialog no longer closes the drawer. --- application/single_app/config.py | 2 +- .../src/components/chat/AssistantMarkdown.tsx | 34 +- .../components/chat/ConversationDrawer.tsx | 7 +- .../src/components/chat/DrawerAssets.tsx | 164 +++--- .../src/components/chat/InlineAudioPlayer.tsx | 57 ++- .../components/chat/InlineMediaFallback.tsx | 16 +- .../src/components/chat/MediaGallery.tsx | 140 ++++++ .../v2_ui/src/components/chat/MediaTiles.tsx | 166 +++++++ .../v2_ui/src/components/chat/MediaViewer.tsx | 294 +++++++++++ .../src/components/chat/useMediaDownload.ts | 47 ++ application/v2_ui/src/lib/inlineMedia.ts | 10 + application/v2_ui/src/lib/mediaDownload.ts | 180 +++++++ application/v2_ui/src/lib/mediaGallery.ts | 465 ++++++++++++++++++ .../features/V2_COLLABORATION_UX.md | 5 + .../V2_INLINE_MEDIA_AND_AGENT_MESSAGES.md | 4 + .../features/V2_MEDIA_GALLERIES_AND_VIEWER.md | 158 ++++++ docs/explanation/features/index.md | 1 + docs/explanation/release_notes.md | 29 ++ docs/reference/chat-controls.md | 16 +- .../test_v2_inline_media_players.py | 5 +- .../test_v2_media_gallery_logic.mjs | 306 ++++++++++++ .../test_v2_media_gallery_wiring.py | 157 ++++++ functional_tests/test_v2_rich_rendering.py | 9 +- ui_tests/test_v2_collaboration_ux.py | 6 +- ui_tests/test_v2_media_galleries.py | 410 +++++++++++++++ 25 files changed, 2594 insertions(+), 94 deletions(-) create mode 100644 application/v2_ui/src/components/chat/MediaGallery.tsx create mode 100644 application/v2_ui/src/components/chat/MediaTiles.tsx create mode 100644 application/v2_ui/src/components/chat/MediaViewer.tsx create mode 100644 application/v2_ui/src/components/chat/useMediaDownload.ts create mode 100644 application/v2_ui/src/lib/mediaDownload.ts create mode 100644 application/v2_ui/src/lib/mediaGallery.ts create mode 100644 docs/explanation/features/V2_MEDIA_GALLERIES_AND_VIEWER.md create mode 100644 functional_tests/test_v2_media_gallery_logic.mjs create mode 100644 functional_tests/test_v2_media_gallery_wiring.py create mode 100644 ui_tests/test_v2_media_galleries.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 7778bc66a..3a890675e 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.258" +VERSION = "0.261.260" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/v2_ui/src/components/chat/AssistantMarkdown.tsx b/application/v2_ui/src/components/chat/AssistantMarkdown.tsx index 817631593..ed6a2d790 100644 --- a/application/v2_ui/src/components/chat/AssistantMarkdown.tsx +++ b/application/v2_ui/src/components/chat/AssistantMarkdown.tsx @@ -1,6 +1,6 @@ // AssistantMarkdown.tsx // Markdown rendering for assistant output: prose, citations, masks, maths, diagrams, charts, -// image proposals, and inline audio, video and image cards. +// image proposals, inline audio, video and image cards, and galleries of images and clips. // // Split out of MessageList so that "what a message looks like" is separable from "how the // thread is laid out". The renderer carries the whole substitution pipeline and the fence @@ -35,8 +35,10 @@ import { InlineImageCard } from './InlineImageCard'; import { InlineImageProposal } from './InlineImageProposal'; import { InlineVideoCard } from './InlineVideoCard'; import { MathDisplay, MathInline } from './MathBlock'; +import { MediaGallery, MediaGalleryTile } from './MediaGallery'; import { MermaidDiagram } from './MermaidDiagram'; import { inlineMediaKind, inlineMediaTitle, safeMarkdownHref, safeMediaUrl } from '../../lib/inlineMedia'; +import { readMediaGallery, readMediaTileIndex, rehypeMediaGallery } from '../../lib/mediaGallery'; import { IMAGE_PROPOSAL_LANGUAGE, INLINE_CHART_LANGUAGE, @@ -240,8 +242,12 @@ function Markdown({ // A link to an audio or video file plays in place instead of navigating away; // every other link renders as before. `node` is the hast element, read for the - // link's plain text and kept off the DOM element. + // link's plain text and kept off the DOM element. A clip in a gallery is a tile. a: ({ href, children, node, ...props }) => { + const tile = readMediaTileIndex(node); + if (tile !== null) { + return ; + } const kind = inlineMediaKind(href); const source = safeMediaUrl(href); if (kind && source) { @@ -258,7 +264,27 @@ function Markdown({ ); }, - img: ({ src, alt }) => , + img: ({ src, alt, node }) => { + const tile = readMediaTileIndex(node); + return tile === null ? : ; + }, + + // A run of images and clips, which rehypeMediaGallery wrapped in a div of figures. + // Markdown produces no other div, but any other is rendered as it was. + div: ({ children, node, ...props }) => { + const items = readMediaGallery(node); + return items ? {children} :
{children}
; + }, + figure: ({ children, node: _node, ...props }) => ( +
+ {children} +
+ ), + figcaption: ({ children }) => ( +
+ {renderTokens(children)} +
+ ), // A diagram, chart or image proposal replaces the whole code block, so the //
 wrapper markdown puts around it is dropped: leaving it would box
@@ -360,7 +386,7 @@ function Markdown({
         >
             
                 {content}
diff --git a/application/v2_ui/src/components/chat/ConversationDrawer.tsx b/application/v2_ui/src/components/chat/ConversationDrawer.tsx
index 18d31acc5..d16bb0aff 100644
--- a/application/v2_ui/src/components/chat/ConversationDrawer.tsx
+++ b/application/v2_ui/src/components/chat/ConversationDrawer.tsx
@@ -199,7 +199,7 @@ function DocumentsMode() {
                 documents={generated.documents}
                 error={generated.error}
             />
-            
+            
         
     ) : null;
 
@@ -267,13 +267,14 @@ export function ConversationDrawer() {
         ),
     );
 
-    // Close on Escape, matching the dismissal behaviour of the rest of the app.
+    // Close on Escape, matching the dismissal behaviour of the rest of the app. A dialog opened
+    // from the drawer, such as the media viewer or a document preview, takes that Escape itself.
     useEffect(() => {
         if (!drawerMode) {
             return;
         }
         const onKeyDown = (event: KeyboardEvent) => {
-            if (event.key === 'Escape') {
+            if (event.key === 'Escape' && !document.querySelector('[role="dialog"][aria-modal="true"]')) {
                 setDrawerMode(null);
             }
         };
diff --git a/application/v2_ui/src/components/chat/DrawerAssets.tsx b/application/v2_ui/src/components/chat/DrawerAssets.tsx
index ce10a9108..56ec7c724 100644
--- a/application/v2_ui/src/components/chat/DrawerAssets.tsx
+++ b/application/v2_ui/src/components/chat/DrawerAssets.tsx
@@ -4,11 +4,12 @@
 // Generated lists the documents agents created in a shared conversation, with Download where the
 // workspace's own download rules allow it and a preview for Markdown. Media gathers every image,
 // video and audio clip the thread shows, including signed links an action fetched from a remote
-// service, which are otherwise hard to find again in a long thread.
+// service, which are otherwise hard to find again in a long thread. Images and clips are tiles,
+// three to a row, that open one viewer stepping through them all; recordings are players, each
+// with Download and a button that scrolls to its message.
 
-import { useCallback, useEffect, useState } from 'react';
-import { createPortal } from 'react-dom';
-import { Download, Eye, FileText, Film, ImageOff, Loader2, Music } from 'lucide-react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
+import { Download, Eye, FileText, Loader2 } from 'lucide-react';
 import { ApiError } from '../../lib/apiClient';
 import {
     fetchCollaborationGeneratedDocument,
@@ -16,16 +17,15 @@ import {
     type GeneratedDocument,
 } from '../../lib/collaboration';
 import { saveBlob } from '../../lib/endpoints';
-import { resolveImageSource } from '../../lib/images';
 import type { ConversationMediaItem } from '../../lib/conversationMedia';
 import { useChatStore } from '../../stores/chatStore';
 import { toast } from '../../stores/toastStore';
 import { Modal } from '../ui/Modal';
 import { PlainMarkdown } from '../ui/PlainMarkdown';
 import { GlassButton } from '../ui/primitives';
-import { ImageLightbox } from './ImageLightbox';
 import { InlineAudioPlayer } from './InlineAudioPlayer';
-import { InlineVideoCard } from './InlineVideoCard';
+import { MediaViewer, type MediaViewerItem } from './MediaViewer';
+import { ImageTile, VideoTile } from './MediaTiles';
 
 /** Largest Markdown file previewed in the app; anything bigger is better read downloaded. */
 const PREVIEW_MAX_BYTES = 2 * 1024 * 1024;
@@ -245,82 +245,108 @@ export function GeneratedDocumentsSection({
     );
 }
 
-function ImageTile({ item, onOpen }: { item: ConversationMediaItem; onOpen: () => void }) {
-    const [failed, setFailed] = useState(false);
+function MediaGroup({ label, count, children }: { label: string; count: number; children: React.ReactNode }) {
     return (
-        
  • - -
  • +
    +

    + {label} + {count} +

    + {children} +
    ); } -function PlayerDialog({ item, onClose }: { item: ConversationMediaItem; onClose: () => void }) { - return ( - - {item.kind === 'video' ? ( - - ) : ( - - )} - +export function MediaSection({ + items, + onLocate, +}: { + items: ConversationMediaItem[]; + /** Scrolls the thread to the message an item appears in. */ + onLocate?: (messageId: string) => void; +}) { + const [viewing, setViewing] = useState(null); + const images = useMemo(() => items.filter((item) => item.kind === 'image'), [items]); + const videos = useMemo(() => items.filter((item) => item.kind === 'video'), [items]); + const recordings = useMemo(() => items.filter((item) => item.kind === 'audio'), [items]); + // One viewer steps through the images and then the clips, in the order the tiles show them. + const visual = useMemo( + () => [...images, ...videos].map(({ kind, src, title, messageId }) => ({ + kind: kind === 'video' ? 'video' : 'image', + src, + title, + messageId, + })), + [images, videos], ); -} -export function MediaSection({ items }: { items: ConversationMediaItem[] }) { - const [viewing, setViewing] = useState(null); if (items.length === 0) { return null; } - const images = items.filter((item) => item.kind === 'image'); - const recordings = items.filter((item) => item.kind !== 'image'); - const imageSource = viewing?.kind === 'image' ? resolveImageSource(viewing.src) : null; return (
    Media - {images.length > 0 && ( -
      - {images.map((item) => ( - setViewing(item)} /> - ))} -
    - )} - {recordings.length > 0 && ( -
      - {recordings.map((item) => ( -
    • - -
    • - ))} -
    - )} - {viewing && imageSource && createPortal( - // Portalled: the drawer's backdrop filter would otherwise confine a fixed overlay to it. - setViewing(null)} />, - document.body, +
    + {images.length > 0 && ( + +
      + {images.map((item, index) => ( +
    • + setViewing(index)} + className="aspect-square" + /> +
    • + ))} +
    +
    + )} + {videos.length > 0 && ( + +
      + {videos.map((item, index) => ( +
    • + setViewing(images.length + index)} + className="aspect-square" + /> +
    • + ))} +
    +
    + )} + {recordings.length > 0 && ( + +
      + {recordings.map((item) => ( +
    • + onLocate(item.messageId) : undefined} + /> +
    • + ))} +
    +
    + )} +
    + {viewing !== null && visual[viewing] && ( + setViewing(null)} + onLocate={onLocate ? (item) => item.messageId && onLocate(item.messageId) : undefined} + /> )} - {viewing && viewing.kind !== 'image' && setViewing(null)} />}
    ); } diff --git a/application/v2_ui/src/components/chat/InlineAudioPlayer.tsx b/application/v2_ui/src/components/chat/InlineAudioPlayer.tsx index 20bcb8ee0..53a41e85b 100644 --- a/application/v2_ui/src/components/chat/InlineAudioPlayer.tsx +++ b/application/v2_ui/src/components/chat/InlineAudioPlayer.tsx @@ -3,24 +3,50 @@ // // Collapsed, it is a single line: play/pause, the recording's title, elapsed and total time, and // a thin progress line. Pressing play (or the chevron) expands it to add a seek slider, stop, -// volume and mute, and playback speed. It stays in the message rather than floating over the -// thread, so a briefing that cites several recordings keeps each one beside the text about it. +// volume and mute, playback speed, download and a link to open it in a new tab. It stays in the +// message rather than floating over the thread, so a briefing that cites several recordings +// keeps each one beside the text about it. The conversation drawer lists recordings with the +// same player, adding a button that scrolls to the message the recording is in. // // Built from , + + {onLocate && ( + + )} )} diff --git a/application/v2_ui/src/components/chat/InlineMediaFallback.tsx b/application/v2_ui/src/components/chat/InlineMediaFallback.tsx index f9e186d7d..7bf99dafd 100644 --- a/application/v2_ui/src/components/chat/InlineMediaFallback.tsx +++ b/application/v2_ui/src/components/chat/InlineMediaFallback.tsx @@ -1,19 +1,25 @@ // InlineMediaFallback.tsx -// What an inline audio or video player turns into when its media cannot be played here. +// What an inline audio or video player, or the media viewer, turns into when its media cannot +// be played or shown here. // // A signed link that has expired, a host the page's content security policy does not allow, or // a format the browser cannot decode all end the same way: the link is still useful, so it is // shown as a link with a short reason rather than as a broken player. import { ExternalLink } from 'lucide-react'; -import { safeMediaUrl, type InlineMediaKind } from '../../lib/inlineMedia'; +import { safeMediaUrl } from '../../lib/inlineMedia'; -export function InlineMediaFallback({ kind, src, title }: { kind: InlineMediaKind; src: string; title: string }) { +const WHAT: Record<'audio' | 'video' | 'image', string> = { + audio: 'This recording can\u2019t be played here.', + video: 'This video can\u2019t be played here.', + image: 'This image can\u2019t be shown here.', +}; + +export function InlineMediaFallback({ kind, src, title }: { kind: 'audio' | 'video' | 'image'; src: string; title: string }) { return ( - This {kind === 'audio' ? 'recording' : 'video'} can't be played here. The link may have expired or its - host may not be allowed. + {WHAT[kind]} The link may have expired or its host may not be allowed. void; +} + +const GalleryContext = createContext(null); + +function describe(items: readonly MediaGalleryItem[]): string { + const images = items.filter((item) => item.kind === 'image').length; + const clips = items.length - images; + const parts = [ + images > 0 ? `${images} ${images === 1 ? 'image' : 'images'}` : '', + clips > 0 ? `${clips} ${clips === 1 ? 'video' : 'videos'}` : '', + ].filter(Boolean); + return `Gallery: ${parts.join(' and ')}`; +} + +export function MediaGallery({ items, children }: { items: readonly MediaGalleryItem[]; children: ReactNode }) { + const [viewing, setViewing] = useState<{ index: number; startAt?: number } | null>(null); + const open = useCallback((index: number, startAt?: number) => setViewing({ index, startAt }), []); + const state = useMemo(() => ({ items, open }), [items, open]); + const viewerItems = useMemo( + () => items.map(({ kind, src, title, detail }) => ({ kind, src, title, detail })), + [items], + ); + + return ( + +
    + {children} +
    + {viewing && ( + setViewing({ index })} + onClose={() => setViewing(null)} + /> + )} +
    + ); +} + +function GalleryClip({ item, onExpand }: { item: MediaGalleryItem; onExpand: (startAt?: number) => void }) { + const videoRef = useRef(null); + const [playing, setPlaying] = useState(false); + const [failed, setFailed] = useState(false); + + const expand = () => { + const video = videoRef.current; + const reached = video?.currentTime ?? 0; + video?.pause(); + onExpand(reached > 0 ? reached : undefined); + }; + + return ( + + {failed ? ( + + + + ) : playing ? ( + + + ) : ( + setPlaying(true)} + className="aspect-[4/3]" + /> + )} + + + ); +} + +/** One tile of the gallery around it, by its position there. */ +export function MediaGalleryTile({ index }: { index: number }) { + const gallery = useContext(GalleryContext); + const item = gallery?.items[index]; + if (!gallery || !item) { + return null; + } + if (item.kind === 'image') { + return ( + gallery.open(index)} + className="aspect-[4/3]" + /> + ); + } + return gallery.open(index, startAt)} />; +} diff --git a/application/v2_ui/src/components/chat/MediaTiles.tsx b/application/v2_ui/src/components/chat/MediaTiles.tsx new file mode 100644 index 000000000..dde36bd69 --- /dev/null +++ b/application/v2_ui/src/components/chat/MediaTiles.tsx @@ -0,0 +1,166 @@ +// MediaTiles.tsx +// The tiles an image or a video clip shows as, in a gallery in a reply and in the Media section +// of the conversation drawer. +// +// An image tile is the picture, cropped to the tile. A clip's tile is its first frame with its +// length and a play mark. That frame is fetched only once the tile is on screen, so opening a +// long thread full of clips does not start loading every one of them. A tile whose image or +// clip will not load says so, and still opens the viewer, which explains why and links to it. + +import { useEffect, useRef, useState, type RefObject } from 'react'; +import { clsx } from 'clsx'; +import { ImageOff, Play, VideoOff } from 'lucide-react'; +import { resolveImageSource } from '../../lib/images'; +import { formatMediaTime, posterFrameUrl, safeMediaUrl } from '../../lib/inlineMedia'; + +const UNAVAILABLE_HINT = 'This could not be loaded. Its link may have expired, or its host may not be allowed.'; + +/** True once the element is on screen, and from then on. */ +function useSeen(ref: RefObject): boolean { + const [seen, setSeen] = useState(() => typeof IntersectionObserver === 'undefined'); + useEffect(() => { + const element = ref.current; + if (seen || !element) { + return undefined; + } + const observer = new IntersectionObserver((entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + setSeen(true); + } + }, { rootMargin: '200px' }); + observer.observe(element); + return () => observer.disconnect(); + }, [ref, seen]); + return seen; +} + +/** What a tile shows when its image or clip will not load. */ +export function MediaUnavailable({ kind }: { kind: 'image' | 'video' }) { + const Icon = kind === 'image' ? ImageOff : VideoOff; + return ( + + + ); +} + +export function ImageTile({ + src, + title, + label, + onOpen, + className, +}: { + src: string; + title: string; + /** The button's accessible name, which says what pressing it does. */ + label: string; + onOpen: () => void; + /** Sizing, such as the tile's aspect ratio. */ + className?: string; +}) { + const [failed, setFailed] = useState(false); + const source = resolveImageSource(src); + return ( + + ); +} + +export function VideoTile({ + src, + title, + label, + onOpen, + className, +}: { + src: string; + title: string; + label: string; + onOpen: () => void; + className?: string; +}) { + const ref = useRef(null); + const seen = useSeen(ref); + const [duration, setDuration] = useState(0); + const [failed, setFailed] = useState(false); + return ( + + ); +} diff --git a/application/v2_ui/src/components/chat/MediaViewer.tsx b/application/v2_ui/src/components/chat/MediaViewer.tsx new file mode 100644 index 000000000..f6a2805c2 --- /dev/null +++ b/application/v2_ui/src/components/chat/MediaViewer.tsx @@ -0,0 +1,294 @@ +// MediaViewer.tsx +// A large view of one image or clip from a set, stepping to the others with the arrow keys or +// the buttons at either side. +// +// Opened from a gallery in a reply, where the set is that gallery, and from the Media section of +// the conversation drawer, where it is every image and clip in the conversation. A clip starts +// playing when it opens, and stepping away stops it. An image can be viewed at its actual size. +// Download saves the file when its host lets the page read it and opens it in a new tab when it +// does not. From the drawer, Show in conversation closes the viewer and scrolls to the message. +// +// Built on Modal, so Escape and a click outside close it, Tab stays inside it, and focus returns +// to the tile that opened it. + +import { useCallback, useEffect, useState } from 'react'; +import { clsx } from 'clsx'; +import { + ChevronLeft, + ChevronRight, + Download, + ExternalLink, + Loader2, + LocateFixed, + Maximize2, + Minimize2, + X, +} from 'lucide-react'; +import { resolveImageSource } from '../../lib/images'; +import { claimPlayback, safeMediaUrl } from '../../lib/inlineMedia'; +import { openMediaInNewTab } from '../../lib/mediaDownload'; +import { toast } from '../../stores/toastStore'; +import { Modal } from '../ui/Modal'; +import { InlineMediaFallback } from './InlineMediaFallback'; +import { useMediaDownload } from './useMediaDownload'; + +export interface MediaViewerItem { + kind: 'image' | 'video'; + /** An image source as resolveImageSource accepts it, or a clip's URL. */ + src: string; + title: string; + /** A second line, such as an image's alt text when a caption gave the title. */ + detail?: string; + /** The message the item appears in, for Show in conversation. */ + messageId?: string; +} + +const HEADER_BUTTON = + 'inline-flex shrink-0 items-center justify-center rounded-lg p-1.5 text-text-3 transition-colors ' + + 'hover:bg-surface-2 hover:text-text-1 disabled:cursor-not-allowed disabled:opacity-50'; +const SIDE_BUTTON = + 'absolute top-1/2 z-10 flex size-10 -translate-y-1/2 items-center justify-center rounded-full bg-black/55 ' + + 'text-white ring-1 ring-white/30 transition-colors hover:bg-black/75 aria-disabled:cursor-default ' + + 'aria-disabled:opacity-30 aria-disabled:hover:bg-black/55'; + +function ViewerStage({ + item, + actualSize, + onToggleZoom, + startAt, +}: { + item: MediaViewerItem; + actualSize: boolean; + onToggleZoom: () => void; + startAt?: number; +}) { + const [failed, setFailed] = useState(false); + + if (item.kind === 'image') { + const source = resolveImageSource(item.src); + if (!source || failed) { + return ; + } + return ( + {item.detail setFailed(true)} + className={clsx( + 'rounded-lg', + actualSize ? 'max-w-none cursor-zoom-out' : 'max-h-full max-w-full cursor-zoom-in object-contain', + )} + /> + ); + } + + if (failed) { + return ; + } + return ( +
    +
    +

    + {item.title} +

    +

    + {kindLabel} + {count > 1 && ` \u00b7 ${position + 1} of ${count}`} +

    +
    + {item.kind === 'image' && ( + + )} + + {item.kind === 'video' ? ( +
    + + + ) : ( + + )} + {onLocate && item.messageId && ( + + )} + +
    + ); + + return ( + +
    +
    + setActualSize((value) => !value)} + startAt={position === openedAt ? startAt : undefined} + /> +
    + {count > 1 && ( + <> + + + + )} +
    + {item.detail && ( +

    {item.detail}

    + )} +
    + ); +} diff --git a/application/v2_ui/src/components/chat/useMediaDownload.ts b/application/v2_ui/src/components/chat/useMediaDownload.ts new file mode 100644 index 000000000..9b23a6522 --- /dev/null +++ b/application/v2_ui/src/components/chat/useMediaDownload.ts @@ -0,0 +1,47 @@ +// useMediaDownload.ts +// Download for an image, clip or recording, reporting the outcome as a toast. +// +// The work is in lib/mediaDownload.ts. This adds what the person sees: that the file was saved, +// that it opened in a new tab because its host does not allow a direct download, or that the +// browser blocked that tab, with a button to open it by hand. + +import { useCallback, useState } from 'react'; +import { downloadMedia, openMediaInNewTab, type MediaKind } from '../../lib/mediaDownload'; +import { toast } from '../../stores/toastStore'; + +const SAVED: Record = { + image: 'Image saved.', + video: 'Video saved.', + audio: 'Recording saved.', +}; + +export function useMediaDownload() { + const [busy, setBusy] = useState(false); + + const download = useCallback(async (kind: MediaKind, src: string, title: string) => { + setBusy(true); + try { + const outcome = await downloadMedia(kind, src, title); + if (outcome === 'saved') { + toast.success(SAVED[kind]); + } else if (outcome === 'opened') { + toast.info('Its host does not allow a direct download, so it opened in a new tab. Save it from there.'); + } else { + toast.info('The browser blocked the new tab.', { + label: 'Open', + onAct: () => { + openMediaInNewTab(kind, src); + }, + }); + } + } catch (error) { + toast.error( + error instanceof Error && error.message ? `Download failed. ${error.message}` : 'Download failed.', + ); + } finally { + setBusy(false); + } + }, []); + + return { download, busy }; +} diff --git a/application/v2_ui/src/lib/inlineMedia.ts b/application/v2_ui/src/lib/inlineMedia.ts index 4dbee46ba..a5ccb2bb6 100644 --- a/application/v2_ui/src/lib/inlineMedia.ts +++ b/application/v2_ui/src/lib/inlineMedia.ts @@ -104,6 +104,16 @@ export function formatMediaTime(seconds: number): string { return hours > 0 ? `${hours}:${String(minutes).padStart(2, '0')}:${secs}` : `${minutes}:${secs}`; } +/** + * A clip's address for a still tile: a media fragment asking for the frame just after the start. + * + * Without one, some browsers leave a clip that has loaded only its metadata blank. The fragment + * never reaches the server, so a signed link's signature is untouched. + */ +export function posterFrameUrl(url: string | null): string | undefined { + return url ? `${url.split('#')[0]}#t=0.1` : undefined; +} + let activeMedia: HTMLMediaElement | null = null; /** diff --git a/application/v2_ui/src/lib/mediaDownload.ts b/application/v2_ui/src/lib/mediaDownload.ts new file mode 100644 index 000000000..23f4908c7 --- /dev/null +++ b/application/v2_ui/src/lib/mediaDownload.ts @@ -0,0 +1,180 @@ +// mediaDownload.ts +// Saving an image, clip or recording a conversation shows, and opening one in a new tab. +// +// Media in a reply is often a signed link to another host. The page can save it directly only +// when that host lets the page read it (CORS). When it does not, the file is opened in a new tab +// instead, where the browser's own viewer can save it. A host refuses quickly, so the new tab +// usually still counts as part of the click; when the browser blocks it anyway, the caller is +// told, so it can offer a link to open it by hand. + +import { CREDENTIALS_MODE } from './apiClient'; +import { saveBlob } from './endpoints'; +import { decodeImageDataUri, openImageInNewTab, resolveImageSource } from './images'; +import { safeMediaUrl } from './inlineMedia'; + +export type MediaKind = 'image' | 'video' | 'audio'; + +/** Saved to downloads, opened in a new tab instead, or refused a new tab by the browser. */ +export type MediaDownloadOutcome = 'saved' | 'opened' | 'blocked'; + +/** How long the host has to start answering before the file is opened in a new tab instead. */ +const RESPONSE_TIMEOUT_MS = 4000; + +/** Characters a file name cannot contain on Windows, plus path separators and controls. */ +// eslint-disable-next-line no-control-regex +const UNSAFE_FILENAME_CHARS = /[<>:"/\\|?*\u0000-\u001f]/g; +const HAS_EXTENSION = /\.[a-z0-9]{2,5}$/i; + +const MIME_EXTENSIONS: Record = { + 'audio/aac': 'aac', + 'audio/flac': 'flac', + 'audio/mp4': 'm4a', + 'audio/mpeg': 'mp3', + 'audio/ogg': 'ogg', + 'audio/wav': 'wav', + 'audio/wave': 'wav', + 'audio/webm': 'weba', + 'audio/x-wav': 'wav', + 'image/gif': 'gif', + 'image/jpeg': 'jpg', + 'image/png': 'png', + 'image/svg+xml': 'svg', + 'image/webp': 'webp', + 'video/mp4': 'mp4', + 'video/ogg': 'ogv', + 'video/quicktime': 'mov', + 'video/webm': 'webm', +}; +const DEFAULT_EXTENSIONS: Record = { image: 'png', video: 'mp4', audio: 'mp3' }; + +/** The file is not there to save: the host answered without it, or its data could not be read. */ +class MediaUnavailableError extends Error { + constructor(message: string) { + super(message); + this.name = 'MediaUnavailableError'; + } +} + +function cleanFileName(value: string): string { + return value + .replace(UNSAFE_FILENAME_CHARS, ' ') + .replace(/\s+/g, ' ') + .trim() + .replace(/^[. ]+/, '') + .slice(0, 80) + .replace(/[. ]+$/, ''); +} + +/** + * The name a file is saved under: the file name at the end of its URL when that has an + * extension, otherwise its title with an extension for its type. + */ +export function mediaFileName(src: string, title: string, kind: MediaKind, blobType = ''): string { + const url = safeMediaUrl(src); + let fromUrl = ''; + if (url) { + try { + fromUrl = cleanFileName(decodeURIComponent(new URL(url).pathname.split('/').pop() ?? '')); + } catch { + fromUrl = ''; + } + } + if (fromUrl && HAS_EXTENSION.test(fromUrl)) { + return fromUrl; + } + const base = cleanFileName(title) || fromUrl || kind; + const extension = MIME_EXTENSIONS[blobType.split(';')[0].trim().toLowerCase()] ?? DEFAULT_EXTENSIONS[kind]; + return `${base}.${extension}`; +} + +/** The address to fetch a file from, and whether it is the app's own authenticated endpoint. */ +function fetchTarget(kind: MediaKind, src: string): { url: string; credentials: RequestCredentials } | null { + if (kind === 'image') { + const source = resolveImageSource(src); + if (!source || source.kind === 'data-uri') { + return null; + } + if (source.kind === 'endpoint') { + return { url: source.src, credentials: CREDENTIALS_MODE }; + } + } + const url = safeMediaUrl(src); + if (!url) { + return null; + } + // Another host gets no cookies: a signed link carries its own authority. + const sameOrigin = new URL(url).origin === window.location.origin; + return { url, credentials: sameOrigin ? 'same-origin' : 'omit' }; +} + +async function fetchMediaBlob(kind: MediaKind, src: string): Promise { + if (kind === 'image') { + const source = resolveImageSource(src); + if (source?.kind === 'data-uri') { + const blob = decodeImageDataUri(source.src); + if (!blob) { + throw new MediaUnavailableError('The image data could not be read.'); + } + return blob; + } + } + const target = fetchTarget(kind, src); + if (!target) { + throw new TypeError('The file has no address the browser can fetch.'); + } + const controller = new AbortController(); + const timer = window.setTimeout(() => controller.abort(), RESPONSE_TIMEOUT_MS); + let response: Response; + try { + response = await fetch(target.url, { credentials: target.credentials, signal: controller.signal }); + } finally { + // Only the wait for an answer is limited. A large clip may take longer to arrive. + window.clearTimeout(timer); + } + if (!response.ok) { + throw new MediaUnavailableError( + `The file could not be fetched (${response.status}). Its link may have expired.`, + ); + } + return response.blob(); +} + +/** Open a file in a new browser tab. False when the browser blocked the tab. */ +export function openMediaInNewTab(kind: MediaKind, src: string): boolean { + if (kind === 'image') { + const source = resolveImageSource(src); + return source ? openImageInNewTab(source) : false; + } + const url = safeMediaUrl(src); + if (!url) { + return false; + } + // Severed by hand: passing `noopener` makes window.open return null even when it succeeds. + const opened = window.open(url, '_blank'); + if (opened) { + opened.opener = null; + } + return Boolean(opened); +} + +/** + * Save a file to the user's downloads, or open it in a new tab when its host will not let the + * page read it. + * + * Throws when the host answered but had no file to give, as when a signed link has expired: + * a new tab would only show the same error. + */ +export async function downloadMedia(kind: MediaKind, src: string, title: string): Promise { + let blob: Blob; + try { + blob = await fetchMediaBlob(kind, src); + } catch (error) { + if (error instanceof MediaUnavailableError) { + throw error; + } + // Refused by CORS, unreachable, or too slow to answer. + return openMediaInNewTab(kind, src) ? 'opened' : 'blocked'; + } + saveBlob(blob, mediaFileName(src, title, kind, blob.type)); + return 'saved'; +} diff --git a/application/v2_ui/src/lib/mediaGallery.ts b/application/v2_ui/src/lib/mediaGallery.ts new file mode 100644 index 000000000..5a4d9e56b --- /dev/null +++ b/application/v2_ui/src/lib/mediaGallery.ts @@ -0,0 +1,465 @@ +// mediaGallery.ts +// Lays out a run of images and video clips in a reply as a gallery, three to a row. +// +// Agents that pull evidence from a remote service tend to answer with one image or clip per +// line, often under a one-line caption, and sometimes a dozen in a row. Shown one under another +// at full width, they push the rest of the reply far down the thread. This rehype plugin finds +// those runs on the parsed tree and wraps each one in a gallery the renderer shows as tiles. +// +// It works on the parsed tree for the same reason rehypeRichBlockIndex does: whether an image +// sits alone on its line depends on paragraphs, hard and soft line breaks and list items, all of +// which the parser has already decided. Each image and clip keeps its own img or a element +// inside the gallery, so react-markdown's URL transform, which runs after every rehype plugin, +// still applies to it, and the renderer still decides what a source may be. +// +// What becomes a gallery: +// - In the body of a reply or a block quote, two or more images or clips in a row, each alone +// on its line. A short line of text directly above a single image or clip, in the same +// paragraph, becomes that tile's caption. A line ending in a colon that introduces several +// images or clips stays text above the gallery. +// - In a list item, the images and clips that end the item: a clip linked after the item's +// last sentence, and images or clips on the lines below it. One is enough, so every item of +// an evidence list reads the same way. +// +// A recording never joins a gallery: it plays from its own player bar, beside the text that +// describes it. Nothing is added that the markdown did not already contain. + +import type { Element, ElementContent, Properties, Root, RootContent } from 'hast'; +import { toText } from 'hast-util-to-text'; +import { resolveImageSource } from './images'; +import { inlineMediaKind, inlineMediaTitle, safeMediaUrl } from './inlineMedia'; + +/** Set on a gallery's wrapper. The value is the number of tiles. */ +export const MEDIA_GALLERY_PROPERTY = 'dataMediaGallery'; +/** Set on each image or link inside a gallery: its position among the gallery's tiles. */ +export const MEDIA_TILE_PROPERTY = 'dataMediaTile'; + +/** The fewest images and clips that make a gallery in a reply's body. A single one keeps its card. */ +export const MIN_BODY_GALLERY_ITEMS = 2; +/** The longest line taken as a caption. Anything longer reads as prose and stays text. */ +export const CAPTION_MAX_LENGTH = 160; + +export type GalleryMediaKind = 'image' | 'video'; + +export interface MediaGalleryItem { + kind: GalleryMediaKind; + /** The image's src or the clip's href, as react-markdown's URL transform left it. */ + src: string; + /** The tile's caption, else the image's alt text or the clip's link text. */ + title: string; + /** The alt or link text, when a caption line supplied the title. Empty otherwise. */ + detail: string; +} + +/** Elements that end a run of lines: everything markdown renders as a block of its own. */ +const BLOCK_TAGS = new Set([ + 'blockquote', 'div', 'figure', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'ol', 'p', 'pre', 'section', 'table', 'ul', +]); +/** Elements whose children can hold paragraphs and lists, and so galleries. */ +const CONTAINER_TAGS = new Set(['blockquote', 'li', 'ol', 'section', 'ul']); +/** Punctuation ending the sentence that a clip is linked after, at the end of a list item's line. */ +const SENTENCE_END = /[.!?:;)\]\u2014\u2013-]$/; +/** Citation, mask and maths placeholders, which a plain-text title must not show raw. */ +const PLACEHOLDER_PATTERN = /\u27E6(cite|mask|math):\d+\u27E7/g; + +type Content = RootContent | ElementContent; + +interface Paragraph { + /** False for a tight list item's text, which markdown leaves unwrapped. */ + wrapped: boolean; + properties: Properties; +} + +interface Line { + paragraph: Paragraph; + nodes: ElementContent[]; +} + +type Piece = { kind: 'line'; line: Line } | { kind: 'node'; node: Content }; + +interface GalleryEntry { + media: Element; + caption: ElementContent[] | null; +} + +function isBlank(node: Content): boolean { + return node.type === 'comment' || (node.type === 'text' && !node.value.trim()); +} + +function isInlineContent(node: Content): node is ElementContent { + return node.type !== 'doctype' && !(node.type === 'element' && BLOCK_TAGS.has(node.tagName)); +} + +/** Plain text for a title: placeholders stood in for and whitespace collapsed. */ +function plainText(value: string): string { + return value + .replace(PLACEHOLDER_PATTERN, (_match, kind: string) => (kind === 'mask' ? '[masked]' : '')) + .replace(/\s+/g, ' ') + .replace(/ ([.,;:!?])/g, '$1') + .trim(); +} + +function lineText(nodes: readonly ElementContent[]): string { + return plainText(toText({ type: 'root', children: [...nodes] })); +} + +/** Whether a node is an image or clip a gallery can hold, and which. */ +export function galleryMediaKind(node: Content | undefined): GalleryMediaKind | null { + if (node?.type !== 'element') { + return null; + } + if (node.tagName === 'img') { + const src = node.properties?.src; + // A data URI is emptied by react-markdown's URL transform after this plugin runs, and a + // path the app does not serve images from never loads: both keep their usual rendering. + return typeof src === 'string' && !/^\s*data:/i.test(src) && resolveImageSource(src) ? 'image' : null; + } + if (node.tagName === 'a') { + return inlineMediaKind(node.properties?.href) === 'video' ? 'video' : null; + } + return null; +} + +/** Whether any image, or any link to audio or video, appears among the nodes. */ +function containsMedia(nodes: readonly Content[]): boolean { + return nodes.some((node) => node.type === 'element' && ( + node.tagName === 'img' + || (node.tagName === 'a' && inlineMediaKind(node.properties?.href) !== null) + || containsMedia(node.children) + )); +} + +function splitLines(nodes: readonly ElementContent[]): ElementContent[][] { + const lines: ElementContent[][] = [[]]; + for (const node of nodes) { + if (node.type === 'element' && node.tagName === 'br') { + lines.push([]); + } else { + lines[lines.length - 1].push(node); + } + } + return lines; +} + +/** A line's nodes without the blank space at either end. */ +function trimLine(nodes: readonly ElementContent[]): ElementContent[] { + let start = 0; + let end = nodes.length; + while (start < end && isBlank(nodes[start])) { + start += 1; + } + while (end > start && isBlank(nodes[end - 1])) { + end -= 1; + } + const trimmed = nodes.slice(start, end); + const first = trimmed[0]; + if (first?.type === 'text') { + trimmed[0] = { ...first, value: first.value.replace(/^\s+/, '') }; + } + const last = trimmed[trimmed.length - 1]; + if (last?.type === 'text') { + trimmed[trimmed.length - 1] = { ...last, value: last.value.replace(/\s+$/, '') }; + } + return trimmed; +} + +/** The images and clips of a line that holds nothing else, or null. */ +function lineMedia(nodes: readonly ElementContent[]): Element[] | null { + const media: Element[] = []; + for (const node of nodes) { + if (isBlank(node)) { + continue; + } + if (node.type !== 'element' || !galleryMediaKind(node)) { + return null; + } + media.push(node); + } + return media.length > 0 ? media : null; +} + +/** Whether a line could caption the single image or clip on the line below it. */ +function isCaptionLine(nodes: readonly ElementContent[]): boolean { + if (containsMedia(nodes)) { + return false; + } + const text = lineText(nodes); + return text.length > 0 && text.length <= CAPTION_MAX_LENGTH; +} + +function endsWithColon(nodes: readonly ElementContent[]): boolean { + return lineText(nodes).endsWith(':'); +} + +/** A caption line's nodes, trimmed, without the colon that labelled the image below it. */ +function captionContent(nodes: readonly ElementContent[]): ElementContent[] { + const trimmed = trimLine(nodes); + const last = trimmed[trimmed.length - 1]; + if (last?.type === 'text' && last.value.endsWith(':')) { + trimmed[trimmed.length - 1] = { ...last, value: last.value.slice(0, -1).replace(/\s+$/, '') }; + } + return trimmed; +} + +/** A list item line's text, and the images or clips linked after its last sentence, or null. */ +function splitTrailingMedia(nodes: readonly ElementContent[]): { text: ElementContent[]; media: Element[] } | null { + let start = nodes.length; + for (let index = nodes.length - 1; index >= 0; index -= 1) { + const node = nodes[index]; + if (isBlank(node)) { + continue; + } + if (!galleryMediaKind(node)) { + break; + } + start = index; + } + const media: Element[] = []; + for (const node of nodes.slice(start)) { + if (node.type === 'element') { + media.push(node); + } + } + const text = nodes.slice(0, start); + // Linked mid-sentence ("compare [this clip](a.mp4) with [that one](b.mp4)") stays where it is. + if (media.length === 0 || !SENTENCE_END.test(lineText(text))) { + return null; + } + return { text: trimLine(text), media }; +} + +/** The text a tile is called by when no caption line names it: alt text, or the link's text. */ +function ownTitle(media: Element, kind: GalleryMediaKind): string { + if (kind === 'image') { + return plainText(String(media.properties?.alt ?? '')); + } + const href = String(media.properties?.href ?? ''); + return plainText(inlineMediaTitle(toText(media), safeMediaUrl(href) ?? href)); +} + +function buildGallery(entries: readonly GalleryEntry[]): Element { + return { + type: 'element', + tagName: 'div', + properties: { [MEDIA_GALLERY_PROPERTY]: entries.length }, + children: entries.map(({ media, caption }, index): Element => { + const kind = galleryMediaKind(media) ?? 'image'; + media.properties = { ...media.properties, [MEDIA_TILE_PROPERTY]: index }; + const captionNodes: ElementContent[] = caption + ? captionContent(caption) + : [{ type: 'text', value: ownTitle(media, kind) }]; + const hasCaption = captionNodes.some((node) => !isBlank(node)); + return { + type: 'element', + tagName: 'figure', + properties: { dataMediaKind: kind }, + children: hasCaption + ? [media, { type: 'element', tagName: 'figcaption', properties: {}, children: captionNodes }] + : [media], + }; + }), + }; +} + +/** A container's children as lines of its paragraphs, and every other node as it is. */ +function readPieces(children: readonly Content[], listItem: boolean): Piece[] { + const pieces: Piece[] = []; + let inline: ElementContent[] = []; + const addLines = (nodes: readonly ElementContent[], paragraph: Paragraph) => { + for (const line of splitLines(nodes)) { + pieces.push({ kind: 'line', line: { paragraph, nodes: line } }); + } + }; + const flushInline = () => { + // A tight list item's text is not wrapped in a paragraph; it is read as one all the same. + if (inline.some((node) => !isBlank(node))) { + addLines(inline, { wrapped: false, properties: {} }); + } else { + pieces.push(...inline.map((node): Piece => ({ kind: 'node', node }))); + } + inline = []; + }; + for (const child of children) { + if (child.type === 'element' && child.tagName === 'p') { + flushInline(); + addLines(child.children, { wrapped: true, properties: child.properties }); + } else if (listItem && isInlineContent(child)) { + inline.push(child); + } else { + flushInline(); + pieces.push({ kind: 'node', node: child }); + } + } + flushInline(); + return pieces; +} + +/** Put the pieces back together, rebuilding each paragraph from the lines it still has. */ +function rebuild(pieces: readonly Piece[]): Content[] { + const children: Content[] = []; + let index = 0; + while (index < pieces.length) { + const piece = pieces[index]; + if (piece.kind === 'node') { + children.push(piece.node); + index += 1; + continue; + } + const { paragraph } = piece.line; + const joined: ElementContent[] = []; + while (index < pieces.length) { + const current = pieces[index]; + if (current.kind !== 'line' || current.line.paragraph !== paragraph) { + break; + } + if (joined.length > 0) { + joined.push({ type: 'element', tagName: 'br', properties: {}, children: [] }); + } + joined.push(...current.line.nodes); + index += 1; + } + const content = trimLine(joined); + if (!content.some((node) => !isBlank(node))) { + continue; + } + if (paragraph.wrapped) { + children.push({ type: 'element', tagName: 'p', properties: paragraph.properties, children: content }); + } else { + children.push(...content); + } + } + return children; +} + +/** Wrap the container's runs of images and clips in galleries. Leaves it untouched if there are none. */ +function regroup(parent: Root | Element, listItem: boolean): void { + const minimum = listItem ? 1 : MIN_BODY_GALLERY_ITEMS; + const sequence = readPieces(parent.children, listItem); + const pieces: Piece[] = []; + let entries: GalleryEntry[] = []; + let consumed: Piece[] = []; + let grouped = false; + + const flush = (): boolean => { + const formed = entries.length >= minimum; + if (formed) { + pieces.push({ kind: 'node', node: buildGallery(entries) }); + } else { + pieces.push(...consumed); + } + entries = []; + consumed = []; + return formed; + }; + + for (let index = 0; index < sequence.length; index += 1) { + const piece = sequence[index]; + if (piece.kind === 'node') { + if (isBlank(piece.node)) { + // Space between blocks neither ends a run nor shows once a run becomes a gallery. + (entries.length > 0 ? consumed : pieces).push(piece); + continue; + } + grouped = flush() || grouped; + pieces.push(piece); + continue; + } + + const { line } = piece; + const media = lineMedia(line.nodes); + if (media) { + entries.push(...media.map((node) => ({ media: node, caption: null }))); + consumed.push(piece); + continue; + } + + // A list item's text describes the item, so it is never taken as one tile's caption. + const next = sequence[index + 1]; + if (!listItem && next?.kind === 'line' && next.line.paragraph === line.paragraph && isCaptionLine(line.nodes)) { + const captioned = lineMedia(next.line.nodes); + // "Stills from both cameras:" over two images introduces them; over one, it labels it. + const after = sequence[index + 2]; + const introducesMore = endsWithColon(line.nodes) && after?.kind === 'line' + && after.line.paragraph === line.paragraph && lineMedia(after.line.nodes) !== null; + if (captioned?.length === 1 && !introducesMore) { + entries.push({ media: captioned[0], caption: line.nodes }); + consumed.push(piece, next); + index += 1; + continue; + } + } + + grouped = flush() || grouped; + const trailing = listItem ? splitTrailingMedia(line.nodes) : null; + if (trailing) { + pieces.push({ kind: 'line', line: { paragraph: line.paragraph, nodes: trailing.text } }); + entries.push(...trailing.media.map((node) => ({ media: node, caption: null }))); + consumed.push({ kind: 'line', line: { paragraph: line.paragraph, nodes: trailing.media } }); + continue; + } + pieces.push(piece); + } + grouped = flush() || grouped; + + if (grouped) { + parent.children = rebuild(pieces) as typeof parent.children; + } +} + +function regroupTree(parent: Root | Element): void { + for (const child of parent.children) { + if (child.type === 'element' && CONTAINER_TAGS.has(child.tagName)) { + regroupTree(child); + } + } + if (parent.type === 'root' || parent.tagName === 'blockquote') { + regroup(parent, false); + } else if (parent.tagName === 'li') { + regroup(parent, true); + } +} + +/** Wrap each run of images and clips in a gallery. */ +export function rehypeMediaGallery() { + return (tree: Root) => { + regroupTree(tree); + }; +} + +/** A media element's position in its gallery, or null when it is not in one. */ +export function readMediaTileIndex(node: Element | undefined): number | null { + const value = node?.properties?.[MEDIA_TILE_PROPERTY]; + return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : null; +} + +/** The tiles of a gallery this plugin built, in order, or null for any other element. */ +export function readMediaGallery(node: Element | undefined): MediaGalleryItem[] | null { + if (node?.tagName !== 'div' || node.properties?.[MEDIA_GALLERY_PROPERTY] === undefined) { + return null; + } + const items: MediaGalleryItem[] = []; + for (const figure of node.children) { + if (figure.type !== 'element' || figure.tagName !== 'figure') { + continue; + } + const media = figure.children.find( + (child): child is Element => child.type === 'element' && readMediaTileIndex(child) !== null, + ); + if (!media) { + continue; + } + const caption = figure.children.find( + (child): child is Element => child.type === 'element' && child.tagName === 'figcaption', + ); + const kind: GalleryMediaKind = media.tagName === 'img' ? 'image' : 'video'; + const own = ownTitle(media, kind); + const title = (caption ? plainText(toText(caption)) : '') || own || (kind === 'image' ? 'Image' : 'Video'); + items.push({ + kind, + src: String((kind === 'image' ? media.properties?.src : media.properties?.href) ?? ''), + title, + detail: own && own !== title ? own : '', + }); + } + return items; +} diff --git a/docs/explanation/features/V2_COLLABORATION_UX.md b/docs/explanation/features/V2_COLLABORATION_UX.md index 188473773..7e5316733 100644 --- a/docs/explanation/features/V2_COLLABORATION_UX.md +++ b/docs/explanation/features/V2_COLLABORATION_UX.md @@ -134,6 +134,11 @@ by the file extension, as the inline players are). Each file appears once. Code spans, fully masked messages and messages replaced by a workflow reply are skipped, as is anything a person typed as plain text. An image whose signed link has expired shows as unavailable. +Since 0.261.260 the section groups media into Images, Videos and Audio. Images and clips are tiles +that open one viewer, which steps through all of them and can scroll to an item's message. +Recordings are players with Download. See +[V2 Media Galleries and Viewer](V2_MEDIA_GALLERIES_AND_VIEWER.md). + ### Files | File | Change | diff --git a/docs/explanation/features/V2_INLINE_MEDIA_AND_AGENT_MESSAGES.md b/docs/explanation/features/V2_INLINE_MEDIA_AND_AGENT_MESSAGES.md index 97119b327..28ff6c778 100644 --- a/docs/explanation/features/V2_INLINE_MEDIA_AND_AGENT_MESSAGES.md +++ b/docs/explanation/features/V2_INLINE_MEDIA_AND_AGENT_MESSAGES.md @@ -20,6 +20,10 @@ usually as short-lived signed links from an OpenAPI action. The V2 chat now pres - Media that cannot play (an expired link, a host the policy does not allow, an unsupported format) falls back to a short notice and a link. +Since 0.261.260, two or more images or clips in a row show as a gallery of tiles, three to a row, +with one viewer for all of them, and recordings can be downloaded. See +[V2 Media Galleries and Viewer](V2_MEDIA_GALLERIES_AND_VIEWER.md). + Messages an agent posts into a conversation through the Simple Chat action (`add_conversation_message`) are marked as agent-authored markdown and render like the agent's replies, so a briefing posted to a group conversation keeps its headings, tables, links, images and diff --git a/docs/explanation/features/V2_MEDIA_GALLERIES_AND_VIEWER.md b/docs/explanation/features/V2_MEDIA_GALLERIES_AND_VIEWER.md new file mode 100644 index 000000000..c08dae356 --- /dev/null +++ b/docs/explanation/features/V2_MEDIA_GALLERIES_AND_VIEWER.md @@ -0,0 +1,158 @@ +# V2 Media Galleries and Viewer + +Implemented in version: **0.261.260** + +## Overview + +Agents that pull evidence from a remote service answer with many images and clips: a dozen +camera stills and a couple of clips in one reply, each under a one-line caption, or a timeline +where every entry links a clip and shows a still. Until this version each image rendered as a +full-width card and each clip as a full-width player, so one such reply filled several screens. +The conversation drawer listed clips and recordings as plain text rows. + +This version lays media out as compact tiles and adds one viewer for all of it: + +- **Galleries in replies.** Two or more images or clips in a row become a gallery of tiles, three + to a row, each with its caption underneath. In a list, the clip and still at the end of each + item sit side by side under the item's text. +- **Clips play in their tile.** A clip tile shows the first frame, its length and a play mark. + Pressing it plays the clip at tile size. **View larger** opens it in the viewer from the same + point. +- **Media viewer.** Opening a tile shows the image or clip large. **Previous** and **Next**, or + the arrow keys, move through the gallery. The viewer also offers Download and Open in a new + tab, and shows the image at its actual size on request. +- **Drawer Media section.** The Documents drawer groups media into **Images**, **Videos** and + **Audio**, with a count for each group. Images and clips are tiles that open one viewer + stepping through all of them, and **Show in conversation** scrolls to the message an item came + from. Recordings are full players with play, stop, seek, volume, speed, Download and Show in + conversation. +- **Downloads.** Recordings, clips and images can be saved. When the file's host does not let + the page read it, the file opens in a new tab instead, where the browser can save it. + +Recordings never join a gallery. A recording plays from its own player bar, next to the text +that describes it. + +### Dependencies + +- The V2 markdown renderer (`react-markdown`, `remark-gfm`, `remark-breaks`) and the inline media + players from [V2 Inline Media and Agent-Posted Messages](V2_INLINE_MEDIA_AND_AGENT_MESSAGES.md). +- The Documents drawer Media section from + [V2 Shared Conversation Experience](V2_COLLABORATION_UX.md). +- `CSP_MEDIA_SRC_ORIGINS` must still allow a media host before its clips and recordings play. +- No new npm, CDN or Python dependency. + +## Technical Specifications + +### Finding galleries in a reply + +`application/v2_ui/src/lib/mediaGallery.ts` is a rehype plugin, `rehypeMediaGallery`. It runs on +the parsed tree for the same reason `rehypeRichBlockIndex` does: paragraphs, hard and soft line +breaks and list items have already been decided there, and a text scanner would get them wrong. + +Lines are split at line breaks, so one paragraph that alternates caption and media lines groups +the same way as separate paragraphs do. + +| Where | What groups | Minimum | +| --- | --- | --- | +| Body of a reply, block quote | Images and clips each alone on a line, in a row. A line of 160 characters or fewer directly above a single image or clip, in the same paragraph, becomes its caption. | 2 | +| List item | The images and clips that end the item: a clip linked after the item's last sentence, and images or clips on the lines below it. The item's text is never taken as a caption. | 1 | + +Rules that keep the grouping predictable: + +- A line ending in a colon that introduces several images stays as text above the gallery + ("Stills from both cameras:"). Over a single image it captions that image, without the colon. +- A clip linked mid-sentence ("compare [this clip](a.mp4) with [that one](b.mp4)") stays in the + sentence. +- Recordings, images in tables, inline `data:` images and paths the app does not serve images + from keep their existing rendering. +- A single image in the body of a reply keeps its full-width card. + +The plugin only moves the `img` and `a` elements that markdown already produced into +`div > figure > figcaption` wrappers. It parses no HTML, raw HTML stays disabled, and +react-markdown's URL transform still runs on every image and link afterwards. Each media element +is marked with `dataMediaTile` (its position in the gallery) and the wrapper with +`dataMediaGallery`. `readMediaGallery` reads the transformed tree back into the gallery's items. +Plain-text titles drop citation and maths placeholders and show masked text as `[masked]`. The +rendered caption keeps its placeholders, so citations and redactions render as usual. + +### Components + +| File | Purpose | +| --- | --- | +| `components/chat/AssistantMarkdown.tsx` | Registers the plugin. `div` renders a gallery, `img` and `a` render a tile when they are in one, `figcaption` substitutes citations, masks and maths. | +| `components/chat/MediaGallery.tsx` | The three-column grid, its viewer, and `MediaGalleryTile`, which renders an image tile or a clip that plays in place with **View larger** | +| `components/chat/MediaTiles.tsx` | `ImageTile`, `VideoTile` (first frame, length, play mark) and the **Unavailable** state | +| `components/chat/MediaViewer.tsx` | The viewer, built on `Modal`: previous and next, arrow keys, actual size, Download, Open in a new tab, Show in conversation | +| `components/chat/useMediaDownload.ts` | Download with a toast for each outcome | +| `components/chat/DrawerAssets.tsx` | The drawer's Images, Videos and Audio groups | +| `components/chat/InlineAudioPlayer.tsx` | Adds Download, and Show in conversation when the drawer lists the recording | +| `components/chat/InlineMediaFallback.tsx` | Also describes an image the viewer cannot show | +| `components/chat/ConversationDrawer.tsx` | Passes Show in conversation to the Media section. Escape no longer closes the drawer while a dialog opened from it is showing. | +| `lib/mediaDownload.ts` | Saving a file, naming it, and opening it in a new tab | +| `lib/inlineMedia.ts` | `posterFrameUrl`, the still-frame address for a clip tile | + +### Loading and playback + +- A clip tile loads only the clip's metadata and first frame, and only once the tile is on screen + (`IntersectionObserver`), so opening a long thread does not fetch every clip in it. The frame + is requested with a `#t=0.1` media fragment, which never reaches the server, so a signed link's + signature is unchanged. +- Starting any clip or recording pauses whichever one was playing, as before. +- Moving to another item in the viewer stops the clip that was showing. + +### Downloads + +`downloadMedia` fetches the file and saves it under the file name at the end of its URL, or under +its title with an extension for its type. The app's own image endpoints are fetched with the +session's credentials. Other hosts are fetched with no cookies, since a signed link carries its +own authority. + +When the host does not let the page read the file (no CORS headers), or does not start answering +within four seconds, the file opens in a new tab and a notice says so. If the browser blocks that +tab, the notice offers **Open**. If the host answers with an error, as for an expired link, the +download reports the error instead of opening a tab that would show the same error. + +### Configuration + +No new settings. Clips and recordings from another host still need its origin in +`CSP_MEDIA_SRC_ORIGINS`, as described in +[V2 Inline Media and Agent-Posted Messages](V2_INLINE_MEDIA_AND_AGENT_MESSAGES.md). + +## Usage + +1. Ask an agent whose actions return images, clips or recordings for them, as before. A reply + with several shows them as a gallery. A reply with one image still shows that image at full + width. +2. Select an image tile to open the viewer, then use **Previous** and **Next** or the arrow keys + to move through the gallery. Select **View at actual size** to inspect detail, and **Download** + to keep a copy. +3. Select a clip tile to play it where it is. Select **View larger** in its corner to continue + watching in the viewer. +4. Open the Documents drawer to find media again in a long conversation. **Images** and + **Videos** open the viewer across every image and clip in the conversation. **Show in + conversation** closes the viewer and scrolls to the message. Under **Audio**, play a recording + in place, or expand it for Download and Show in conversation. + +## Testing and Validation + +| Test | Covers | +| --- | --- | +| `functional_tests/test_v2_media_gallery_logic.mjs` | The real plugin inside react-markdown on caption pairs, alternating lines, image paragraphs, colon labels, evidence lists, tight lists, mid-sentence clips, unsafe and `data:` sources, block quotes, tables, placeholders in captions and the URL transform; file naming and the still-frame address | +| `functional_tests/test_v2_media_gallery_wiring.py` | Runs the Node checks; renderer wiring; no raw HTML; every new media element and link uses `safeMediaUrl` or `resolveImageSource`; drawer groups and Escape guard; download credentials and fallback | +| `functional_tests/test_v2_inline_media_players.py` | Updated for the new image wiring | +| `functional_tests/test_v2_rich_rendering.py` | The pinned rehype plugin list now includes `rehypeMediaGallery` | +| `ui_tests/test_v2_media_galleries.py` | In Chromium with production CSS: three-column layout and captions, evidence-list rows, viewer navigation by buttons and keys, focus returning to the tile, a clip playing in its tile and opening larger, drawer groups, a recording playing and downloading, Show in conversation, Escape closing only the viewer, the new-tab fallback, and an expired link | +| `ui_tests/test_v2_collaboration_ux.py` | The drawer's Generated and Media sections, updated for clip tiles | + +The UI tests play a WebM clip recorded by the browser under test and a WAV built by the test, so +no media files are committed. + +### Known limitations + +- Grouping follows the shape of the markdown. A caption written after its image, or several + images on one line under one caption, is not captioned. +- The viewer steps through one gallery from a reply, or through all images and clips from the + drawer. It does not move between galleries in different replies. +- A host that will not let the page read its files can't be saved directly. The file opens in a + new tab instead, where the browser's own viewer can save it. +- The classic interface is unchanged. diff --git a/docs/explanation/features/index.md b/docs/explanation/features/index.md index 793c6df69..d2c48dcce 100644 --- a/docs/explanation/features/index.md +++ b/docs/explanation/features/index.md @@ -52,6 +52,7 @@ category: Version History ## Versioned Features - [Saved Analyze Results](ANALYZE_RESULTS.md) +- [V2 Media Galleries And Viewer](V2_MEDIA_GALLERIES_AND_VIEWER.md) - [V2 Inline Diagram Editing](V2_INLINE_DIAGRAM_EDITING.md) - [V2 Inline Chart Editing](V2_INLINE_CHART_EDITING.md) - [V2 Inline Image Editing](V2_INLINE_IMAGE_EDITING.md) diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index cde8b127f..6e383ee10 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,35 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.260)** + +#### User Interface Enhancements + +* **Media Galleries in V2 Replies** + * Two or more images or clips in a row now show as a gallery of tiles, three to a row, each with its caption underneath, instead of one full-width card each. A short caption line above an image or clip becomes its caption. In a list, the clip and still at the end of each item sit side by side under the item's text. A single image keeps its full-width card, and recordings keep their player bar. + * A clip tile shows its first frame and length, and plays where it is. **View larger** continues it in the viewer. + * Grouping happens on the parsed markdown: raw HTML stays disabled and react-markdown's URL checks still apply to every image and link. + * (Ref: `mediaGallery.ts`, `MediaGallery.tsx`, `MediaTiles.tsx`, `AssistantMarkdown.tsx`, [V2 Media Galleries and Viewer](features/V2_MEDIA_GALLERIES_AND_VIEWER.md)) + +* **Media Viewer** + * Opening an image or clip shows it large, with **Previous** and **Next** buttons and the arrow keys stepping through the gallery, its position ("3 of 15"), View at actual size, Download and Open in a new tab. Focus returns to the tile when it closes. + * (Ref: `MediaViewer.tsx`, `mediaDownload.ts`, `useMediaDownload.ts`) + +* **Drawer Media Section Groups Images, Videos and Audio** + * The Documents drawer now lists media under **Images**, **Videos** and **Audio** with counts. Clips show as tiles with their first frame instead of text rows, and images and clips open one viewer across all of them, with **Show in conversation** to scroll to the message. + * Recordings play in the drawer, with play, stop, seek, volume, speed, Download and Show in conversation. + * (Ref: `DrawerAssets.tsx`, `ConversationDrawer.tsx`, `InlineAudioPlayer.tsx`) + +* **Download for Recordings, Clips and Images** + * Recordings, clips and images can be saved from the player and the viewer, under the file name in their link. When a host does not let the page read its files, the file opens in a new tab instead, where the browser can save it. An expired link reports that it has expired. + * (Ref: `mediaDownload.ts`, `InlineAudioPlayer.tsx`, `MediaViewer.tsx`) + +#### Bug Fixes + +* **Escape in a Drawer Dialog No Longer Closes the Drawer** + * Pressing Escape in a dialog opened from the conversation drawer, such as a document preview or the media viewer, also closed the drawer. Escape now closes only the dialog, and a second Escape closes the drawer. + * (Ref: `ConversationDrawer.tsx`, `test_v2_media_galleries.py`) + ### **(v0.261.258)** #### User Interface Enhancements diff --git a/docs/reference/chat-controls.md b/docs/reference/chat-controls.md index f9baf2583..46543f927 100644 --- a/docs/reference/chat-controls.md +++ b/docs/reference/chat-controls.md @@ -367,7 +367,21 @@ These controls exist only in the V2 interface, so they are not part of the gener | Mention pills on a message | Show the agent that was asked and the people who were named as pills above the message text, with your own name highlighted. The `@Name` text is removed from what is displayed, including where a reply quotes the message; the stored message keeps it. | Use them to see at a glance who a message was for, the same way a reply shows what it answers. | [`enable_collaborative_conversations`]({{ '/admin/chat/' | relative_url }}) | | Agent activity line | Shows one slim line at the end of the thread for each running request: which agent, who asked, how long it has been running, and what it is doing in plain words, such as "Looking up order status", "Asking Data Analyst" or "Writing the answer". Everyone in the conversation sees it, several can run at once, and it replaces the large "Thinking" bubble in shared conversations. | Use it to know an agent is still working, and on what, without the thread filling with placeholders. | [`enable_collaborative_conversations`]({{ '/admin/chat/' | relative_url }}) | | Generated (Documents drawer) | Lists the documents agents created in this conversation with the SimpleChat upload actions. Markdown opens in a preview, and Download appears only when the workspace's download rules allow it for you. | Use it to open or save a briefing or report an agent produced without hunting for it in a workspace. | [`allow_group_workspace_file_downloads`]({{ '/admin/workspaces/' | relative_url }}) for group documents
    [`allow_personal_workspace_file_downloads`]({{ '/admin/workspaces/' | relative_url }}) for personal documents | -| Media (Documents drawer) | Gathers every image, video and audio clip shown in the conversation, including signed links an action fetched from a remote service. Images open full size; clips and recordings open in a player. | Use it to find a photo or clip again in a long thread. | Always available | +| Media (Documents drawer) | Gathers every image, video and audio clip shown in the conversation, including signed links an action fetched from a remote service, grouped as **Images**, **Videos** and **Audio** with a count for each. Images and clips open one viewer that steps through all of them, with **Show in conversation** to scroll to the message. Recordings play in place and can be downloaded. | Use it to find a photo, clip or recording again in a long thread. | Always available | + +## Media in replies (V2 interface) + +These controls exist only in the V2 interface, so they are not part of the generated inventory above, which is taken from the classic chat page. They keep a reply that returns many images and clips readable: two or more in a row show as tiles, three to a row, instead of one full-width card each. See [V2 media galleries and viewer]({{ '/explanation/features/V2_MEDIA_GALLERIES_AND_VIEWER/' | relative_url }}) for the full description. + +| Control | What it does | Why you would use it | Enabled by | +| --- | --- | --- | --- | +| Gallery image tile | Opens the media viewer on that image, with the rest of the gallery a step away. The tile shows **Unavailable** when its link has expired. | Use it to look at a still or scan at full size without losing your place in the thread. | Always available | +| Gallery clip tile | Shows the clip's first frame and length. Pressing it plays the clip where it is, at tile size, with the browser's own controls. | Use it to glance at a short clip next to the text about it. | Always available; another host's clips need [`CSP_MEDIA_SRC_ORIGINS`]({{ '/explanation/features/V2_INLINE_MEDIA_AND_AGENT_MESSAGES/' | relative_url }}) | +| View larger | Opens a clip in the media viewer, continuing from where it had reached in its tile. | Use it when a clip needs more detail than the tile shows. | Same as the clip tile | +| Media viewer: Previous, Next and arrow keys | Move through the images and clips in the gallery, or through every image and clip when opened from the drawer. The header shows the position, such as "3 of 15". | Use them to review a set of stills and clips in order without closing and reopening each. | Always available | +| Media viewer: View at actual size | Shows an image at its natural size, scrolling to pan; selecting the image does the same. | Use it to read a plate, a document or other fine detail. | Always available | +| Media viewer and recording: Download | Saves the image, clip or recording under the file name in its link. When its host does not allow a direct download, it opens in a new tab instead, where the browser can save it. | Use it to keep a copy of evidence outside the conversation. | Always available | +| Show in conversation | Closes the viewer and scrolls to the message the item appears in, highlighting it briefly. Offered from the Documents drawer. | Use it to read what an agent said about an image, clip or recording you found in the drawer. | Always available | ## Attached prompts (V2 interface) diff --git a/functional_tests/test_v2_inline_media_players.py b/functional_tests/test_v2_inline_media_players.py index fc835f608..c5a8e3015 100644 --- a/functional_tests/test_v2_inline_media_players.py +++ b/functional_tests/test_v2_inline_media_players.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ Functional test for V2 inline media players, image cards and agent-posted group messages. -Version: 0.261.222 +Version: 0.261.260 Implemented in: 0.261.222 This test ensures that: @@ -158,7 +158,8 @@ def test_markdown_renderer_wires_inline_media_safely(): assert "a: ({ href, children, node, ...props }) =>" in markdown assert "const kind = inlineMediaKind(href);" in markdown assert "" in markdown - assert "img: ({ src, alt }) => " in markdown + # Since 0.261.260 an image or clip in a gallery renders as a tile; every other image keeps its card. + assert "return tile === null ? : ;" in markdown assert "from 'rehype-raw'" not in markdown, "raw HTML must stay off" assert "allowDangerousHtml" not in markdown and "urlTransform=" not in markdown, "keep react-markdown's default URL transform" diff --git a/functional_tests/test_v2_media_gallery_logic.mjs b/functional_tests/test_v2_media_gallery_logic.mjs new file mode 100644 index 000000000..d87d34e67 --- /dev/null +++ b/functional_tests/test_v2_media_gallery_logic.mjs @@ -0,0 +1,306 @@ +// test_v2_media_gallery_logic.mjs +// Version: 0.261.260 +// Implemented in: 0.261.260 +// Runs the real V2 gallery plugin inside the real react-markdown pipeline (remark-gfm and +// remark-breaks, as the chat renders replies) over the shapes agents actually answer with: a +// caption line over each image or clip, one paragraph alternating caption and media, images on +// their own, and evidence lists that link a clip after each item's sentence with a still below. +// Also checks what must not group (a single image, a recording, a link mid-sentence, a table, +// unsafe or inline data URLs), that react-markdown's URL transform still runs on every grouped +// image and clip, and the download file naming and still-frame helpers. + +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; +import './test_support/tsResolve.mjs'; + +const hadWindow = 'window' in globalThis; +const previousWindow = globalThis.window; +globalThis.window = { location: { href: 'https://chat.example.test/chat', origin: 'https://chat.example.test' } }; + +const require = createRequire(new URL('../application/v2_ui/package.json', import.meta.url)); +const React = require('react'); +const { renderToStaticMarkup } = require('react-dom/server'); +const load = (name) => import(pathToFileURL(require.resolve(name)).href); + +const checks = []; +const check = (name, run) => checks.push([name, run]); + +try { + const { default: Markdown, defaultUrlTransform } = await load('react-markdown'); + const { default: remarkGfm } = await load('remark-gfm'); + const { default: remarkBreaks } = await load('remark-breaks'); + const { readMediaGallery, rehypeMediaGallery } = await import('../application/v2_ui/src/lib/mediaGallery.ts'); + const { mediaFileName } = await import('../application/v2_ui/src/lib/mediaDownload.ts'); + const { posterFrameUrl } = await import('../application/v2_ui/src/lib/inlineMedia.ts'); + + const MEDIA = 'https://media.example.test/media'; + + /** Render as the chat does, recording what the renderer reads off each gallery. */ + function render(markdown, urlTransform = defaultUrlTransform) { + const galleries = []; + const html = renderToStaticMarkup(React.createElement(Markdown, { + remarkPlugins: [remarkGfm, remarkBreaks], + rehypePlugins: [rehypeMediaGallery], + urlTransform, + components: { + div: ({ node, children, ...props }) => { + const items = readMediaGallery(node); + if (items) { + galleries.push(items); + } + return React.createElement('div', props, children); + }, + }, + }, markdown)); + return { html, galleries }; + } + + const count = (html, pattern) => (html.match(pattern) ?? []).length; + const captions = (html) => [...html.matchAll(/
    ([\s\S]*?)<\/figcaption>/g)].map((match) => match[1]); + + check('caption paragraphs over each image or clip become one captioned gallery', () => { + const { html, galleries } = render([ + 'Here is what the records returned.', + '', + 'Lobby camera still \u2014 record R-1; October 4, 13:38. ', + `![Lobby camera at 13:38](${MEDIA}/lobby-still-1.png?exp=1&sig=a)`, + '', + 'Badge photo \u2014 record R-1. ', + `![Badge photo](${MEDIA}/badge-1.png?exp=1&sig=b)`, + '', + 'Loading dock clip \u2014 record R-2; October 4, 13:37. ', + `[Open video: Loading dock camera clip at 13:37 (8 s)](${MEDIA}/dock-clip-2.mp4?exp=1&sig=c)`, + '', + 'Receipt scan \u2014 record R-3. ', + `![Receipt R-3](${MEDIA}/receipt-3.png)`, + '', + 'Initial call \u2014 recording C-1. ', + `[Open audio: Call audio C-1](${MEDIA}/call-1.wav?exp=1&sig=d)`, + ].join('\n')); + assert.equal(galleries.length, 1, html); + assert.deepEqual(galleries[0].map((item) => [item.kind, item.title, item.detail]), [ + ['image', 'Lobby camera still \u2014 record R-1; October 4, 13:38.', 'Lobby camera at 13:38'], + ['image', 'Badge photo \u2014 record R-1.', 'Badge photo'], + ['video', 'Loading dock clip \u2014 record R-2; October 4, 13:37.', 'Loading dock camera clip at 13:37 (8 s)'], + ['image', 'Receipt scan \u2014 record R-3.', 'Receipt R-3'], + ]); + assert.equal(galleries[0][0].src, `${MEDIA}/lobby-still-1.png?exp=1&sig=a`, 'a signed link keeps its query'); + assert.match(html, /^

    Here is what the records returned\.<\/p>\s*

    /); + assert.equal(count(html, /
    /g), 3); + assert.equal(count(html, /
    /g), 1); + assert.deepEqual([...html.matchAll(/data-media-tile="(\d+)"/g)].map((match) => match[1]), ['0', '1', '2', '3']); + // The recording keeps its own player, after the gallery, with its caption beside it. + assert.match(html, /<\/div>\s*

    Initial call \u2014 recording C-1\.\s*Open audio: Call audio C-1<\/a><\/p>$/); + }); + + check('one paragraph alternating caption and media lines groups the same way', () => { + const { html, galleries } = render([ + 'Lobby camera still \u2014 record R-1.', + `![Lobby camera at 13:38](${MEDIA}/lobby-still-1.png)`, + 'Loading dock clip \u2014 record R-2.', + `[Open video: Loading dock camera clip](${MEDIA}/dock-clip-2.mp4)`, + 'Receipt scan \u2014 record R-3.', + `![Receipt R-3](${MEDIA}/receipt-3.png)`, + 'Initial call \u2014 recording C-1.', + `[Open audio: Call audio C-1](${MEDIA}/call-1.wav)`, + ].join('\n')); + assert.equal(galleries.length, 1); + assert.deepEqual(galleries[0].map((item) => item.title), [ + 'Lobby camera still \u2014 record R-1.', 'Loading dock clip \u2014 record R-2.', 'Receipt scan \u2014 record R-3.', + ]); + assert.match(html, /<\/div>\s*

    Initial call \u2014 recording C-1\.\s* { + const { html, galleries } = render([ + '## Images', + '', + `![Identity document scan](${MEDIA}/document.png)`, + '', + `![Arrival photo](${MEDIA}/arrival.png)`, + ].join('\n')); + assert.equal(galleries.length, 1); + assert.deepEqual(galleries[0].map((item) => [item.title, item.detail]), [ + ['Identity document scan', ''], ['Arrival photo', ''], + ]); + assert.deepEqual(captions(html), ['Identity document scan', 'Arrival photo']); + assert.match(html, /^

    Images<\/h2>\s*
    /); + }); + + check('a single image keeps its full-size rendering', () => { + const { html, galleries } = render(`Intro.\n\n![Only one](${MEDIA}/one.png)\n\nOutro.`); + assert.equal(galleries.length, 0); + assert.match(html, /

    Only one<\/p>/); + }); + + check('a line ending in a colon over several images introduces them and stays text', () => { + const { html, galleries } = render([ + 'Stills from both cameras:', + `![North camera](${MEDIA}/north.png)`, + `![South camera](${MEDIA}/south.png)`, + 'That is everything.', + ].join('\n')); + assert.equal(galleries.length, 1); + assert.deepEqual(galleries[0].map((item) => item.title), ['North camera', 'South camera']); + assert.match(html, /^

    Stills from both cameras:<\/p>\s*

    [\s\S]*<\/div>\s*

    That is everything\.<\/p>$/); + }); + + check('a label ending in a colon over a single image captions it, without the colon', () => { + const { html, galleries } = render([ + '## Signed documents', + '', + '**Search of room 117** (application A-1; execute by October 19): ', + `![Signed document A-1](${MEDIA}/document-a1.png)`, + '', + '**Search of the vehicle** (application A-2): ', + `![Signed document A-2](${MEDIA}/document-a2.png)`, + ].join('\n')); + assert.equal(galleries.length, 1, html); + assert.deepEqual(galleries[0].map((item) => [item.title, item.detail]), [ + ['Search of room 117 (application A-1; execute by October 19)', 'Signed document A-1'], + ['Search of the vehicle (application A-2)', 'Signed document A-2'], + ]); + assert.deepEqual(captions(html), [ + 'Search of room 117 (application A-1; execute by October 19)', + 'Search of the vehicle (application A-2)', + ]); + }); + + check('an evidence list puts each item\'s clip and still in a row under its text', () => { + const { html, galleries } = render([ + `- **06:31, camera 12:** Gray sedan heading west on Main Street. [Video](${MEDIA}/clip-12-0631.mp4) `, + ` ![Camera 12 still 06:31](${MEDIA}/still-12-0631.png)`, + '', + `- **07:01, camera 12:** Same sedan heading east. [Video](${MEDIA}/clip-12-0701.mp4) `, + ` ![Camera 12 still 07:01](${MEDIA}/still-12-0701.png)`, + ].join('\n')); + assert.equal(galleries.length, 2, html); + for (const gallery of galleries) { + assert.deepEqual(gallery.map((item) => item.kind), ['video', 'image']); + assert.equal(gallery[0].title, 'Video'); + } + assert.match(html, /

  • \s*

    06:31, camera 12:<\/strong> Gray sedan heading west on Main Street\.<\/p>\s*

  • Citation 4410 \u2014 issued 08:40\.<\/strong> A sedan was cited near the inn\.
    /); + assert.match(html, /
  • No media in this item\.<\/li>/); + }); + + check('clips linked mid-sentence stay where they are', () => { + const { html, galleries } = render( + `- Compare [the first clip](${MEDIA}/a.mp4) with [the second](${MEDIA}/b.mp4)`, + ); + assert.equal(galleries.length, 0); + assert.equal(count(html, / { + const { galleries } = render([ + `[Open video: North gate clip](${MEDIA}/north.mp4)`, + `[Watch the clip - South gate](${MEDIA}/south.webm)`, + `[${MEDIA}/east.mov](${MEDIA}/east.mov)`, + ].join('\n')); + assert.deepEqual(galleries[0].map((item) => [item.kind, item.title]), [ + ['video', 'North gate clip'], ['video', 'South gate'], ['video', 'east.mov'], + ]); + }); + + check('unsafe and inline data sources never group', () => { + const { html, galleries } = render([ + '![Bad](javascript:alert(1))', + '![Inline](data:image/png;base64,AAAA)', + '[Open video: Bad clip](javascript:alert(1)//x.mp4)', + `![Not served here](/static/x.png)`, + ].join('\n')); + assert.equal(galleries.length, 0); + assert.doesNotMatch(html, /javascript:|data:image/, 'react-markdown still empties unsafe URLs'); + }); + + check('block quotes group, tables do not', () => { + const quoted = render(`> ![Quoted one](${MEDIA}/q1.png)\n> ![Quoted two](${MEDIA}/q2.png)`); + assert.equal(quoted.galleries.length, 1); + assert.match(quoted.html, /^
    \s*
    /); + const table = render([ + '| A | B |', + '| --- | --- |', + `| ![one](${MEDIA}/t1.png) | ![two](${MEDIA}/t2.png) |`, + ].join('\n')); + assert.equal(table.galleries.length, 0); + }); + + check('titles drop citation and maths placeholders and stand in for masked text', () => { + const { html, galleries } = render([ + 'Camera still \u27E6cite:0\u27E7 \u2014 reviewed by \u27E6mask:1\u27E7.', + `![Still A](${MEDIA}/a.png)`, + 'Second still \u27E6math:0\u27E7.', + `![Still B](${MEDIA}/b.png)`, + ].join('\n')); + assert.deepEqual(galleries[0].map((item) => item.title), [ + 'Camera still \u2014 reviewed by [masked].', 'Second still.', + ]); + // The caption keeps its placeholders, for the renderer to swap for chips and redactions. + assert.match(captions(html)[0], /\u27E6cite:0\u27E7/); + }); + + check('react-markdown\'s URL transform still runs on every grouped image and clip', () => { + const seen = []; + const { html, galleries } = render([ + `![One](${MEDIA}/1.png)`, + `[Open video: Two](${MEDIA}/2.mp4)`, + ].join('\n'), (url, key) => { + seen.push(key); + return `${url}#checked`; + }); + assert.deepEqual(galleries[0].map((item) => item.src), [`${MEDIA}/1.png#checked`, `${MEDIA}/2.mp4#checked`]); + assert.deepEqual(seen.sort(), ['href', 'src']); + assert.match(html, /src="[^"]*1\.png#checked"/); + }); + + check('downloads are named after the file in the URL, else the title', () => { + assert.equal(mediaFileName(`${MEDIA}/call-1.wav?exp=1&sig=d`, 'Call audio C-1', 'audio'), 'call-1.wav'); + assert.equal(mediaFileName(`${MEDIA}/a%20b.mp3`, 'x', 'audio'), 'a b.mp3'); + assert.equal(mediaFileName('https://media.example.test/api/clip/123', 'Dock clip', 'video', 'video/webm'), 'Dock clip.webm'); + assert.equal(mediaFileName('data:image/png;base64,AAAA', 'Lobby: still?', 'image', 'image/png'), 'Lobby still.png'); + assert.equal(mediaFileName(`${MEDIA}/`, '', 'video'), 'video.mp4'); + assert.equal(mediaFileName(`${MEDIA}/..%2F..%2Fevil.mp4`, 'x', 'video'), 'evil.mp4'); + assert.equal(mediaFileName('javascript:alert(1)', 'Clip', 'video', 'text/html'), 'Clip.mp4'); + }); + + check('a still frame is asked for without touching the signed query', () => { + assert.equal(posterFrameUrl(`${MEDIA}/a.mp4?sig=1`), `${MEDIA}/a.mp4?sig=1#t=0.1`); + assert.equal(posterFrameUrl(`${MEDIA}/a.mp4#t=5`), `${MEDIA}/a.mp4#t=0.1`); + assert.equal(posterFrameUrl(null), undefined); + }); + + let failures = 0; + for (const [name, run] of checks) { + try { + run(); + console.log(`PASS ${name}`); + } catch (error) { + failures += 1; + console.error(`FAIL ${name}\n${error.stack}`); + } + } + if (failures) { + process.exitCode = 1; + } +} finally { + if (hadWindow) { + globalThis.window = previousWindow; + } else { + delete globalThis.window; + } +} diff --git a/functional_tests/test_v2_media_gallery_wiring.py b/functional_tests/test_v2_media_gallery_wiring.py new file mode 100644 index 000000000..e71f0ce21 --- /dev/null +++ b/functional_tests/test_v2_media_gallery_wiring.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +""" +Functional test for V2 media galleries, the media viewer and the drawer's Media section. +Version: 0.261.260 +Implemented in: 0.261.260 + +This test ensures that: + - runs of images and clips in a reply group into galleries the way agents actually write them, + downloads are named sensibly and a clip's still frame never touches its signed query (the + real TypeScript runs inside react-markdown in Node: test_v2_media_gallery_logic.mjs); + - the renderer wires the gallery without weakening its XSS boundary: no raw HTML, the default + URL transform, and every image, clip and link the new components render reading its address + through resolveImageSource or safeMediaUrl; + - the conversation drawer groups images, videos and recordings into one viewer and players, + scrolls to an item's message, and stays open when Escape belongs to a dialog it opened; + - recordings can be downloaded, cookies are never sent to another host, and a download opens + the file in a new tab instead of failing when its host will not let the page read it. +""" + +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +V2_SRC = ROOT / "application" / "v2_ui" / "src" +CHAT_DIR = V2_SRC / "components" / "chat" +LIB_DIR = V2_SRC / "lib" + +sys.path.insert(0, str(ROOT / "functional_tests")) + +from test_support.versioning import assert_app_version_at_least # noqa: E402 + +IMPLEMENTED_IN = "0.261.260" +NEW_COMPONENTS = ("MediaGallery.tsx", "MediaTiles.tsx", "MediaViewer.tsx") +MEDIA_ELEMENT = re.compile(r"<(a|audio|video)\b[^>]*?\b(href|src)=\{([^}]*)\}", re.S) +IMAGE_ELEMENT = re.compile(r"]*?\bsrc=\{([^}]*)\}", re.S) + + +def _read(path): + return path.read_text(encoding="utf-8") + + +def test_gallery_logic_runs_in_node(): + """Run the real plugin, download naming and still-frame helpers in Node.""" + print("Testing gallery grouping in Node...") + subprocess.run(["node", str(ROOT / "functional_tests" / "test_v2_media_gallery_logic.mjs")], cwd=ROOT, check=True) + print("Gallery grouping passed!") + + +def test_renderer_wires_galleries_without_raw_html(): + """The plugin regroups existing elements; nothing in the pipeline parses or injects HTML.""" + print("Testing the renderer wiring...") + markdown = _read(CHAT_DIR / "AssistantMarkdown.tsx") + assert "rehypePlugins={[rehypeRichBlockIndex, rehypeMediaGallery, rehypeHighlightSubset]}" in markdown + assert "import { readMediaGallery, readMediaTileIndex, rehypeMediaGallery } from '../../lib/mediaGallery';" in markdown + assert "const items = readMediaGallery(node);" in markdown + assert "{children}" in markdown + assert re.search(r"figcaption: \(\{ children \}\) => \(\s*]*>\s*\{renderTokens\(children\)\}", markdown), ( + "captions must pass through renderTokens so citations and masks never show as raw placeholders" + ) + assert "const tile = readMediaTileIndex(node);" in markdown + assert "from 'rehype-raw'" not in markdown, "raw HTML must stay off" + assert "urlTransform=" not in markdown, "react-markdown's default URL transform must stay in place" + + plugin = _read(LIB_DIR / "mediaGallery.ts") + for sink in ("type: 'raw'", "innerHTML", "dangerouslySetInnerHTML", "fromHtml", "rehype-raw"): + assert sink not in plugin, f"the gallery plugin must not produce markup from text ({sink})" + assert "!/^\\s*data:/i.test(src) && resolveImageSource(src)" in plugin, ( + "only images the app can show, and that react-markdown keeps, may join a gallery" + ) + assert "inlineMediaKind(node.properties?.href) === 'video'" in plugin + print("Renderer wiring passed!") + + +def test_new_media_components_read_addresses_safely(): + """Every image, clip and link the new components render goes through a scheme check.""" + print("Testing media addresses in the new components...") + for name in NEW_COMPONENTS: + text = _read(CHAT_DIR / name) + assert "dangerouslySetInnerHTML" not in text, name + for match in MEDIA_ELEMENT.finditer(text): + assert "safeMediaUrl(" in match.group(3), f"{name}: <{match.group(1)} {match.group(2)}> must use safeMediaUrl" + for match in IMAGE_ELEMENT.finditer(text): + assert match.group(1).strip() == "source.src", f"{name}: must come from resolveImageSource" + assert "resolveImageSource(" in text, name + for anchor in re.finditer(r"]*>", text, re.S): + if 'target="_blank"' in anchor.group(0): + assert 'rel="noopener noreferrer"' in anchor.group(0), f"{name}: a new-tab link must not keep its opener" + + tiles = _read(CHAT_DIR / "MediaTiles.tsx") + assert "src={posterFrameUrl(safeMediaUrl(src))}" in tiles + assert 'preload="metadata"' in tiles and "IntersectionObserver" in tiles, "clip tiles load their frame only once on screen" + viewer = _read(CHAT_DIR / "MediaViewer.tsx") + assert " onLocate(item.messageId) : undefined}" in drawer_assets + assert "" in drawer + assert "event.key === 'Escape' && !document.querySelector('[role=\"dialog\"][aria-modal=\"true\"]')" in drawer + print("Drawer Media section passed!") + + +def test_downloads_never_send_cookies_elsewhere_and_fall_back_to_a_tab(): + """Downloads authenticate only to the app, and open the file when a host refuses the read.""" + print("Testing media downloads...") + download = _read(LIB_DIR / "mediaDownload.ts") + assert "return { url, credentials: sameOrigin ? 'same-origin' : 'omit' };" in download + assert "return { url: source.src, credentials: CREDENTIALS_MODE };" in download + assert "return openMediaInNewTab(kind, src) ? 'opened' : 'blocked';" in download + assert "opened.opener = null;" in download + assert "if (error instanceof MediaUnavailableError) {" in download, "an expired link reports an error, not a new tab" + + player = _read(CHAT_DIR / "InlineAudioPlayer.tsx") + assert "onClick={() => void download('audio', src, title)}" in player + assert "aria-label={`Download ${title}`}" in player + assert player.count('type="range"') == 2, "the player keeps exactly its seek and volume sliders" + print("Media downloads passed!") + + +def test_version_was_incremented(): + assert_app_version_at_least(IMPLEMENTED_IN) + + +if __name__ == "__main__": + tests = [ + test_gallery_logic_runs_in_node, + test_renderer_wires_galleries_without_raw_html, + test_new_media_components_read_addresses_safely, + test_drawer_groups_media_and_guards_escape, + test_downloads_never_send_cookies_elsewhere_and_fall_back_to_a_tab, + test_version_was_incremented, + ] + failures = 0 + for test in tests: + try: + test() + except Exception as exc: # noqa: BLE001 - report every failing check + failures += 1 + print(f"FAILED {test.__name__}: {exc}") + print(f"{len(tests) - failures}/{len(tests)} tests passed") + sys.exit(1 if failures else 0) diff --git a/functional_tests/test_v2_rich_rendering.py b/functional_tests/test_v2_rich_rendering.py index 33b01febc..6b19f8ea5 100644 --- a/functional_tests/test_v2_rich_rendering.py +++ b/functional_tests/test_v2_rich_rendering.py @@ -2,7 +2,7 @@ """ Functional test for V2 TeX, Mermaid and SimpleChart rendering. -Version: 0.261.223 +Version: 0.261.260 Implemented in: 0.261.024 Three kinds of block the application already produces were shown in the V2 chat as raw @@ -386,9 +386,12 @@ def test_rich_fences_are_wired_into_the_renderer(): # Pinned as the exact list rather than a membership check, so adding a plugin is a # deliberate, reviewable change. `rehypeRichBlockIndex` only stamps a number onto fences # that already render as a diagram or chart; it introduces no new markup. + # `rehypeMediaGallery` (0.261.260) only regroups the img and a elements markdown already + # produced into div/figure/figcaption wrappers: it parses no HTML, and react-markdown's URL + # transform still runs on every image and link afterwards (test_v2_media_gallery_logic.mjs). assert ( - "rehypePlugins={[rehypeRichBlockIndex, rehypeHighlightSubset]}" in source - ), "The rehype pipeline must stay limited to the block numberer and the curated highlighter" + "rehypePlugins={[rehypeRichBlockIndex, rehypeMediaGallery, rehypeHighlightSubset]}" in source + ), "The rehype pipeline must stay limited to the block numberer, the media gallery and the curated highlighter" print("Fence wiring test passed!") return True diff --git a/ui_tests/test_v2_collaboration_ux.py b/ui_tests/test_v2_collaboration_ux.py index 1770b27ec..d4d60915d 100644 --- a/ui_tests/test_v2_collaboration_ux.py +++ b/ui_tests/test_v2_collaboration_ux.py @@ -1,7 +1,7 @@ # test_v2_collaboration_ux.py """ UI test for the V2 shared conversation experience. -Version: 0.261.255 +Version: 0.261.260 Implemented in: 0.261.255 This test ensures that, in a shared conversation, a message names the people and the agent it @@ -91,6 +91,10 @@ def handle(self, route): if path.startswith("/captures/"): route.fulfill(status=200, body=PIXEL, content_type="image/png") return + # The drawer's clip tile loads the clip's first frame; an unplayable clip shows as unavailable. + if path.startswith("/clips/"): + route.fulfill(status=404, body="Media not found.", content_type="text/plain") + return if request.method == "POST" and path == f"{base}/typing": route.fulfill(json={"success": True}) return diff --git a/ui_tests/test_v2_media_galleries.py b/ui_tests/test_v2_media_galleries.py new file mode 100644 index 000000000..9544fb7ee --- /dev/null +++ b/ui_tests/test_v2_media_galleries.py @@ -0,0 +1,410 @@ +# test_v2_media_galleries.py +""" +UI test for V2 media galleries, the media viewer and the drawer's Media section. +Version: 0.261.260 +Implemented in: 0.261.260 + +This test ensures that a run of images and clips in a reply is laid out three to a row with each +caption under its tile while a recording keeps its own player; that an evidence list puts each +item's clip and still in a row under the item's text; that a tile opens a viewer that steps +through the gallery with its buttons and the arrow keys and hands focus back to the tile; that a +clip plays in its own tile and opens larger; and that the conversation drawer groups images, +videos and recordings, downloads a recording, steps through every image and clip, scrolls to the +message an item is in, and stays open when Escape closes the viewer. A host that will not let the +page read a file opens it in a new tab instead of failing. Only HTTP boundaries are mocked: the +real MessageList, ConversationDrawer and stores run in Chromium with production CSS, playing a +WebM clip recorded in the browser and a WAV recording built here. + +Build: npm --prefix .\\application\\v2_ui run build -- --outDir ..\\..\\ui_tests\\artifacts\\orchestration-plan-editor +Run: python -m pytest .\\ui_tests\\test_v2_media_galleries.py -q +""" + +import base64 +import io +import re +import wave +from urllib.parse import urlsplit + +import pytest +from playwright.sync_api import expect + +from test_v2_orchestration_plan_editor import ( # noqa: F401 + HARNESS, + ORIGIN, + connect_options, + editor_assets, + editor_browser, +) + +pytestmark = pytest.mark.ui + +CONVERSATION = "media-chat" +MEDIA = f"{ORIGIN}/media" +# Another host, which will not let the page read its files: it shows them, but a fetch of one fails. +REMOTE = "https://media.example.test" +PIXEL = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" +) + + +def silent_wav(seconds=0.6, rate=8000): + buffer = io.BytesIO() + with wave.open(buffer, "wb") as handle: + handle.setnchannels(1) + handle.setsampwidth(1) + handle.setframerate(rate) + handle.writeframes(b"\x80" * int(rate * seconds)) + return buffer.getvalue() + + +WAV = silent_wav() + +RECORD_CLIP = """async () => { + const canvas = document.createElement('canvas'); + canvas.width = 64; + canvas.height = 48; + const context = canvas.getContext('2d'); + const recorder = new MediaRecorder(canvas.captureStream(15), { mimeType: 'video/webm;codecs=vp8' }); + const chunks = []; + recorder.ondataavailable = (event) => chunks.push(event.data); + const stopped = new Promise((resolve) => { recorder.onstop = resolve; }); + let frame = 0; + const paint = () => { context.fillStyle = frame++ % 2 ? '#2a7' : '#a27'; context.fillRect(0, 0, 64, 48); }; + paint(); + const timer = setInterval(paint, 60); + recorder.start(); + await new Promise((resolve) => setTimeout(resolve, 2500)); + recorder.stop(); + clearInterval(timer); + await stopped; + const bytes = new Uint8Array(await new Blob(chunks, { type: 'video/webm' }).arrayBuffer()); + let binary = ''; + for (const byte of bytes) { binary += String.fromCharCode(byte); } + return btoa(binary); +}""" + + +@pytest.fixture(scope="module") +def clip_bytes(editor_browser): + """A short, playable WebM, recorded by the browser under test.""" + page = editor_browser.new_page() + try: + return base64.b64decode(page.evaluate(RECORD_CLIP)) + finally: + page.close() + + +class MediaApi: + """The HTTP boundary: the harness, the app's own media and a remote media host.""" + + def __init__(self, assets, clip): + self.assets = assets + self.clip = clip + self.remote = [] + self.unexpected = [] + + def media(self, route, name): + if name.startswith("missing"): + route.fulfill(status=404, body="Media not found.", content_type="text/plain") + elif name.endswith(".png"): + route.fulfill(status=200, body=PIXEL, content_type="image/png") + elif name.endswith((".mp4", ".webm")): + route.fulfill(status=200, body=self.clip, content_type="video/webm") + elif name.endswith(".wav"): + route.fulfill(status=200, body=WAV, content_type="audio/wav") + else: + return False + return True + + def handle(self, route): + request = route.request + parsed = urlsplit(request.url) + origin = f"{parsed.scheme}://{parsed.netloc}" + name = parsed.path.rsplit("/", 1)[-1] + if origin == REMOTE: + self.remote.append(f"{request.resource_type} {parsed.path}") + # A host without CORS headers makes the page's fetch reject, the same as a network + # failure. Playwright's fulfilled responses always pass CORS, so the read is failed here + # while images and new tabs, which need no CORS, are still served. + if request.resource_type == "fetch": + route.abort() + elif not self.media(route, name): + route.fulfill(status=404) + return + if origin != ORIGIN: + self.unexpected.append(request.url) + route.abort() + return + if request.method == "GET" and parsed.path in self.assets: + route.fulfill(path=str(self.assets[parsed.path])) + return + if parsed.path.startswith("/media/") and self.media(route, name): + return + if parsed.path == "/favicon.ico": + route.fulfill(status=204) + return + self.unexpected.append(f"{request.method} {parsed.path}") + route.fulfill(status=404, json={"error": "Unmocked request"}) + + +@pytest.fixture +def media_ui(editor_browser, editor_assets, clip_bytes): + context = editor_browser.new_context(viewport={"width": 1280, "height": 1000}, accept_downloads=True) + api = MediaApi(editor_assets, clip_bytes) + errors = [] + # On the context, so a file opened in a new tab is served too. + context.route("**/*", api.handle) + page = context.new_page() + page.on("pageerror", lambda error: errors.append(str(error))) + try: + yield page, api + finally: + context.close() + assert not api.unexpected, f"Unexpected browser requests: {api.unexpected}" + assert not errors, errors + + +def reply(message_id, content): + return { + "id": message_id, + "conversation_id": CONVERSATION, + "role": "assistant", + "content": content, + "timestamp": "2026-10-06T16:12:20Z", + "metadata": {}, + } + + +def mount(page, api, messages, mounts, drawer=None): + page.goto(ORIGIN + HARNESS) + page.wait_for_function("() => Boolean(window.OrchHarness)") + for url in api.assets: + if url.endswith(".css"): + page.add_style_tag(url=ORIGIN + url) + page.evaluate( + """(spec) => { + const H = window.OrchHarness; + H.reset(); + H.stores.bootstrap.useBootstrapStore.setState({ data: { + version: '0.261.260', settings: {}, branding: { app_title: 'SimpleChat' }, features: {}, + user: { id: 'ui-user', display_name: 'Pat Reader' }, + scope: { groups: [], public_workspaces: [] }, + catalogs: { models: [], agents: [], prompts: [] }, + } }); + H.stores.chat.useChatStore.setState({ + activeConversationId: spec.conversation, activeConversationKind: 'personal', + conversations: [{ id: spec.conversation, title: 'Evidence review' }], + messagesLoading: false, messagesError: null, streaming: false, streamingContent: '', + streamError: null, thoughts: [], messages: spec.messages, + drawerMode: spec.drawer, metadata: { used_documents: [] }, + }); + for (const [id, name] of spec.mounts) { + H.mount(id, name); + } + }""", + {"conversation": CONVERSATION, "messages": messages, "mounts": mounts, "drawer": drawer}, + ) + + +EVIDENCE = reply("a-1", "\n".join([ + "Here is what the records returned.", + "", + "Lobby camera still \u2014 record R-1; October 4, 13:38. ", + f"![Lobby camera at 13:38]({MEDIA}/lobby-still-1.png?exp=1&sig=a)", + "", + "Badge photo \u2014 record R-1. ", + f"![Badge photo]({MEDIA}/badge-1.png?exp=1&sig=b)", + "", + "Loading dock clip \u2014 record R-2. ", + f"[Open video: Loading dock camera clip (2 s)]({MEDIA}/dock-clip-2.mp4?exp=1&sig=c)", + "", + "Receipt scan \u2014 record R-3. ", + f"![Receipt R-3]({MEDIA}/receipt-3.png)", + "", + "Initial call \u2014 recording C-1. ", + f"[Open audio: Call audio C-1]({MEDIA}/call-1.wav?exp=1&sig=d)", +])) + +TIMELINE = reply("a-2", "\n".join([ + f"- **06:31, camera 12:** Gray sedan heading west on Main Street. [Video]({MEDIA}/clip-12-0631.mp4) ", + f" ![Camera 12 still 06:31]({MEDIA}/still-12-0631.png)", + "", + f"- **07:01, camera 12:** Same sedan heading east. [Video]({MEDIA}/clip-12-0701.mp4) ", + f" ![Camera 12 still 07:01]({MEDIA}/still-12-0701.png)", +])) + + +def box(locator): + value = locator.bounding_box() + assert value, "expected a laid-out element" + return value + + +def test_reply_media_runs_render_as_three_column_galleries(media_ui): + page, api = media_ui + mount(page, api, [EVIDENCE, TIMELINE], [["test-root", "MessageList"]]) + + evidence = page.locator("#message-a-1") + gallery = evidence.get_by_role("group", name="Gallery: 3 images and 1 video") + expect(gallery).to_be_visible() + figures = gallery.locator("figure") + expect(figures).to_have_count(4) + expect(gallery.locator("figcaption")).to_have_text([ + "Lobby camera still \u2014 record R-1; October 4, 13:38.", + "Badge photo \u2014 record R-1.", + "Loading dock clip \u2014 record R-2.", + "Receipt scan \u2014 record R-3.", + ]) + tiles = [box(figures.nth(index).locator("[data-media-tile-kind]").first) for index in range(4)] + assert tiles[0]["y"] == pytest.approx(tiles[1]["y"], abs=1) == pytest.approx(tiles[2]["y"], abs=1), tiles + assert tiles[0]["x"] < tiles[1]["x"] < tiles[2]["x"], tiles + assert tiles[3]["y"] > tiles[0]["y"] + tiles[0]["height"], "the fourth tile starts the next row" + for tile in tiles: + assert 150 <= tile["width"] <= 209, tiles + # Replies with a gallery no longer show the full-width card per image. + expect(evidence.get_by_role("button", name=re.compile("^View the full-size image"))).to_have_count(0) + # The recording keeps its player, after the gallery, with its caption beside it. + expect(evidence.get_by_role("group", name="Audio: Call audio C-1")).to_be_visible() + expect(evidence.get_by_text("Initial call \u2014 recording C-1.")).to_be_visible() + + timeline = page.locator("#message-a-2") + items = timeline.locator("li") + expect(items).to_have_count(2) + for index, sentence in enumerate(["Gray sedan heading west on Main Street.", "Same sedan heading east."]): + item = items.nth(index) + expect(item.locator("p").first).to_contain_text(sentence) + row = item.get_by_role("group", name="Gallery: 1 image and 1 video") + expect(row.get_by_role("button", name="Play video: Video")).to_be_visible() + expect(row.get_by_role("button", name=re.compile("^View image: Camera 12 still"))).to_be_visible() + clip, still = box(row.locator("[data-media-tile-kind]").nth(0)), box(row.locator("[data-media-tile-kind]").nth(1)) + assert clip["y"] == pytest.approx(still["y"], abs=1) and clip["x"] < still["x"], (clip, still) + + +def test_gallery_viewer_steps_through_images_and_clips(media_ui): + page, api = media_ui + mount(page, api, [EVIDENCE], [["test-root", "MessageList"]]) + gallery = page.get_by_role("group", name="Gallery: 3 images and 1 video") + + opener = gallery.get_by_role("button", name="View image: Badge photo \u2014 record R-1.") + opener.click() + viewer = page.get_by_role("dialog") + position = viewer.locator("[data-media-viewer-position]") + expect(viewer.get_by_role("heading", name="Badge photo \u2014 record R-1.")).to_be_visible() + expect(position).to_have_text("Image \u00b7 2 of 4") + expect(viewer.get_by_text("Badge photo", exact=True)).to_be_visible() + expect(viewer.locator("[data-media-viewer] img")).to_have_attribute("src", re.compile(r"/media/badge-1\.png\?exp=1&sig=b$")) + + page.keyboard.press("ArrowRight") + expect(position).to_have_text("Video \u00b7 3 of 4") + clip = viewer.locator("[data-media-viewer] video") + expect(clip).to_have_attribute("src", re.compile(r"/media/dock-clip-2\.mp4\?exp=1&sig=c$")) + page.wait_for_function("() => { const v = document.querySelector('[data-media-viewer] video'); return v && !v.paused && v.currentTime > 0; }") + + viewer.get_by_role("button", name="Next").click() + expect(position).to_have_text("Image \u00b7 4 of 4") + expect(viewer.locator("[data-media-viewer] video")).to_have_count(0) + expect(viewer.get_by_role("button", name="Next")).to_have_attribute("aria-disabled", "true") + page.keyboard.press("ArrowRight") + expect(position).to_have_text("Image \u00b7 4 of 4") + viewer.get_by_role("button", name="Previous").click() + expect(position).to_have_text("Video \u00b7 3 of 4") + + page.keyboard.press("Escape") + expect(page.get_by_role("dialog")).to_have_count(0) + expect(opener).to_be_focused() + + +def test_a_gallery_clip_plays_in_its_tile_and_opens_larger(media_ui): + page, api = media_ui + mount(page, api, [EVIDENCE], [["test-root", "MessageList"]]) + gallery = page.get_by_role("group", name="Gallery: 3 images and 1 video") + + gallery.get_by_role("button", name="Play video: Loading dock clip \u2014 record R-2.").click() + inline = gallery.locator("video[controls]") + expect(inline).to_have_count(1) + page.wait_for_function("() => { const v = document.querySelector('[data-media-gallery] video[controls]'); return v && !v.paused && v.currentTime > 0.2; }") + tile = box(inline) + assert tile["width"] <= 209, tile + + gallery.get_by_role("button", name="View larger: Loading dock clip \u2014 record R-2.").click() + viewer = page.get_by_role("dialog") + expect(viewer.locator("[data-media-viewer-position]")).to_have_text("Video \u00b7 3 of 4") + page.wait_for_function("() => { const v = document.querySelector('[data-media-viewer] video'); return v && !v.paused; }") + assert page.evaluate("() => document.querySelector('[data-media-gallery] video[controls]').paused"), ( + "opening the clip larger pauses it in its tile" + ) + assert box(viewer.locator("[data-media-viewer] video"))["width"] > tile["width"] + + +def test_drawer_groups_media_plays_recordings_and_finds_their_message(media_ui): + page, api = media_ui + mount(page, api, [EVIDENCE], [["test-root", "MessageList"], ["mount-a", "ConversationDrawer"]], drawer="documents") + media = page.get_by_role("region", name="Media") + + expect(media.get_by_role("group", name="Images").get_by_role("button")).to_have_count(3) + expect(media.get_by_role("group", name="Videos").get_by_role("button")).to_have_count(1) + audio = media.get_by_role("group", name="Audio") + expect(audio.get_by_role("group", name="Audio: Call audio C-1")).to_be_visible() + + # A recording plays from the drawer, and downloads under the name in its link. + audio.get_by_role("button", name="Play Call audio C-1").click() + page.wait_for_function("() => [...document.querySelectorAll('[data-drawer-media] audio')].some((a) => !a.paused || a.ended)") + with page.expect_download() as download: + audio.get_by_role("button", name="Download Call audio C-1").click() + assert download.value.suggested_filename == "call-1.wav" + + # The viewer steps through the images, then the clip. + media.get_by_role("button", name="Play video: Loading dock camera clip (2 s)").click() + viewer = page.get_by_role("dialog") + position = viewer.locator("[data-media-viewer-position]") + expect(position).to_have_text("Video \u00b7 4 of 4") + page.keyboard.press("ArrowLeft") + expect(position).to_have_text("Image \u00b7 3 of 4") + expect(viewer.get_by_role("heading", name="Receipt R-3")).to_be_visible() + + # Escape closes the viewer only; a second Escape closes the drawer. + page.keyboard.press("Escape") + expect(page.get_by_role("dialog")).to_have_count(0) + drawer = page.get_by_role("complementary", name="Conversation details") + expect(drawer).to_be_visible() + + # Show in conversation closes the viewer and highlights the message. + media.get_by_role("button", name="View image: Badge photo").click() + page.get_by_role("dialog").get_by_role("button", name="Show in conversation").click() + expect(page.get_by_role("dialog")).to_have_count(0) + expect(page.locator("#message-a-1")).to_have_class(re.compile(r"\bring-2\b")) + + page.keyboard.press("Escape") + expect(drawer).to_have_count(0) + + +def test_media_the_page_cannot_read_opens_in_a_new_tab(media_ui): + page, api = media_ui + remote = reply("a-3", "\n".join([ + f"![Remote still one]({REMOTE}/media/remote-1.png?sig=r1)", + f"![Remote still two]({REMOTE}/media/remote-2.png?sig=r2)", + f"![Expired still]({MEDIA}/missing-3.png)", + ])) + mount(page, api, [remote], [["test-root", "MessageList"], ["mount-b", "Toaster"]]) + gallery = page.get_by_role("group", name="Gallery: 3 images") + + gallery.get_by_role("button", name="View image: Remote still one").click() + viewer = page.get_by_role("dialog") + with page.context.expect_page() as opened: + viewer.get_by_role("button", name="Download Remote still one").click() + tab = opened.value + tab.wait_for_load_state() + assert tab.url == f"{REMOTE}/media/remote-1.png?sig=r1", tab.url + tab.close() + assert "fetch /media/remote-1.png" in api.remote, "the page tried to save the file before opening it" + expect(page.get_by_text("Its host does not allow a direct download")).to_be_visible() + page.keyboard.press("Escape") + + # A link that has expired says so on its tile and in the viewer, and cannot be downloaded. + expired = gallery.get_by_role("button", name="View image: Expired still") + expect(expired).to_contain_text("Unavailable") + expired.click() + viewer = page.get_by_role("dialog") + expect(viewer.get_by_text("This image can\u2019t be shown here.")).to_be_visible() + viewer.get_by_role("button", name="Download Expired still").click() + expect(page.get_by_text("The file could not be fetched (404). Its link may have expired.")).to_be_visible()