From 5dc075fdce21751fcb01563d05d598d41e409da9 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Fri, 4 Sep 2026 09:09:46 -0400 Subject: [PATCH] feat(webv2): show what an image reference costs, and stop stacking 2048px defaults An image reference is patchified like any frame -- width * height / 1024 rows at H3's 16x spatial compression and 2x2 patch -- and those rows are re-attended at EVERY denoising step, with attention quadratic in the sequence length. "Max detail" pins the short edge to 2048 whatever the generation size is, so a 1920x1080 reference normalizes to 3648x2048 and adds 7,296 rows: +19% on a 1344x768 x 124-frame request (~1.8 GiB by the denoise node's 0.25 MiB/row model, ~1.4x the attention work), and +34% on a 768x768 one. "Match generation size" costs ~1,000 rows for the same image. Nothing in the panel said any of this before the queue. Two changes: - Each image reference card now shows the size the graph will encode and the rows it adds, live as the detail or the canvas changes. `resolveMiniMaxH3ReferenceImage` mirrors the backend's `resolve_reference_image_short_edge` + `normalize_reference_image`, banker's rounding included, so the estimate is the real size; its table test is cross-checked against the Python implementation. - The FIRST image reference still defaults to "max" -- upstream's rule, and the primary subject is where the detail earns its cost -- while later ones default to "match". Three 2048px references are what turn a modest surcharge into a doubling, and the marginal value of a supporting reference at seven times the output's pixel density is small. The help text says so, and the per-card row count makes it visible. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01S4B5exWsbC2z2Uu2tA167A --- .../frontend/webv2/public/locales/en.json | 3 +- .../features/video/core/dimensions.test.ts | 45 +++++++++++++ .../src/features/video/core/dimensions.ts | 66 ++++++++++++++++++- .../src/features/video/core/settings.test.ts | 24 +++++++ .../webv2/src/features/video/core/settings.ts | 17 +++++ .../video/ui/VideoReferenceListField.tsx | 34 +++++++++- .../src/features/video/ui/VideoWidgetView.tsx | 1 + 7 files changed, 186 insertions(+), 4 deletions(-) diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 17154e7b0b5..015af92f2fc 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -3458,10 +3458,11 @@ "referenceExtendCapFullDescription": "All 3 video reference slots are in use, so there is no room for the continuity reference this clip needs. Remove a video reference, then drop the clip again.", "referenceExtendHelp": "The generated video is appended to this clip at its End Frame. A linked video reference samples up to the last ~5 seconds before that cutpoint for continuity — never more than the generated clip's own length, since the model discards the overrun at the seam. Its trim re-derives whenever the cutpoints or the frame count change.", "referenceImageCapRace": "All {{max}} image reference slots filled while this one was loading, so it was not added.", + "referenceImageCost": "{{width}}×{{height}} · {{rows}} rows per step", "referenceVideoCapRace": "All {{max}} video reference slots filled while this one was loading, so it was not added.", "referenceFromInitialVideo": "Initial video", "references": "References", - "referencesHelp": "Up to 3 videos and 9 images condition the generation, in order — reordering references changes the result. A video reference can contribute its image track, its soundtrack, or both.", + "referencesHelp": "Up to 3 videos and 9 images condition the generation, in order — reordering references changes the result. A video reference can contribute its image track, its soundtrack, or both. Every reference adds rows to each denoising step, so the first image reference starts at maximum detail and later ones match the generation size.", "removeReference": "Remove reference", "staleReferences": "References are set but the selected model cannot use them." } diff --git a/invokeai/frontend/webv2/src/features/video/core/dimensions.test.ts b/invokeai/frontend/webv2/src/features/video/core/dimensions.test.ts index c291fb8a5a3..d1db267c6aa 100644 --- a/invokeai/frontend/webv2/src/features/video/core/dimensions.test.ts +++ b/invokeai/frontend/webv2/src/features/video/core/dimensions.test.ts @@ -9,6 +9,7 @@ import { MINIMAX_H3_NUM_FRAMES_CHOICES, MINIMAX_H3_NUM_FRAMES_DEFAULT, resolveMiniMaxH3Canvas, + resolveMiniMaxH3ReferenceImage, scaleAndSnapWanDimensions, snapMiniMaxH3NumFrames, snapWanNumFrames, @@ -150,6 +151,50 @@ describe('MiniMax H3 frame counts', () => { }); }); +describe('resolveMiniMaxH3ReferenceImage', () => { + // Cross-checked against the backend's own `resolve_reference_image_short_edge` + + // `normalize_reference_image` (the graph encodes exactly these sizes). + const LANDSCAPE_AREA = 1344 * 768; + const SQUARE_AREA = 768 * 768; + + it.each([ + // source, detail, target area, normalized size, rows + [1920, 1080, 'max', LANDSCAPE_AREA, 3648, 2048, 7296], + [1920, 1080, 'match', LANDSCAPE_AREA, 1344, 768, 1008], + [4032, 3024, 'max', LANDSCAPE_AREA, 2720, 2048, 5440], + [4032, 3024, 'match', LANDSCAPE_AREA, 1184, 896, 1036], + [3024, 4032, 'match', LANDSCAPE_AREA, 896, 1184, 1036], + [1024, 1024, 'max', LANDSCAPE_AREA, 2048, 2048, 4096], + [1920, 1080, 'match', SQUARE_AREA, 1024, 576, 576], + [1024, 1024, 'match', SQUARE_AREA, 768, 768, 576], + ] as const)( + '%sx%s at %s detail normalizes to %sx%s', + (width, height, detail, targetArea, expectedWidth, expectedHeight, expectedRows) => { + expect(resolveMiniMaxH3ReferenceImage(width, height, detail, targetArea)).toEqual({ + dimensions: { height: expectedHeight, width: expectedWidth }, + rows: expectedRows, + }); + } + ); + + it('never scales a match-detail reference above the 2048 rule', () => { + const huge = resolveMiniMaxH3ReferenceImage(4000, 4000, 'match', 4096 * 4096); + + expect(huge?.dimensions).toEqual({ height: 2048, width: 2048 }); + }); + + it('sizes a max-detail reference without a target area', () => { + expect(resolveMiniMaxH3ReferenceImage(1920, 1080, 'max', null)?.rows).toBe(7296); + }); + + it('returns null when match detail has no area to match, or the source is degenerate', () => { + expect(resolveMiniMaxH3ReferenceImage(1920, 1080, 'match', null)).toBeNull(); + expect(resolveMiniMaxH3ReferenceImage(1920, 1080, 'match', 0)).toBeNull(); + expect(resolveMiniMaxH3ReferenceImage(0, 1080, 'max', LANDSCAPE_AREA)).toBeNull(); + expect(resolveMiniMaxH3ReferenceImage(Number.NaN, 1080, 'max', LANDSCAPE_AREA)).toBeNull(); + }); +}); + describe('getVideoDurationSeconds', () => { it('matches the backend n / fps labeling and guards degenerate inputs', () => { expect(getVideoDurationSeconds(124, 24)).toBeCloseTo(5.17, 2); diff --git a/invokeai/frontend/webv2/src/features/video/core/dimensions.ts b/invokeai/frontend/webv2/src/features/video/core/dimensions.ts index 294c5b6bc10..6f671cc2bcb 100644 --- a/invokeai/frontend/webv2/src/features/video/core/dimensions.ts +++ b/invokeai/frontend/webv2/src/features/video/core/dimensions.ts @@ -1,4 +1,9 @@ -import type { MiniMaxH3TargetResolution, VideoAspectRatioId, WanTargetResolution } from './types'; +import type { + MiniMaxH3TargetResolution, + VideoAspectRatioId, + VideoReferenceImageDetail, + WanTargetResolution, +} from './types'; /** * Client-side ports of the backend's video canvas math, so the panel can show @@ -8,6 +13,8 @@ import type { MiniMaxH3TargetResolution, VideoAspectRatioId, WanTargetResolution * ("nearest" rounding — the node default; the other modes are workflow-only). * - MiniMax H3: `resolve_canvas_size` in `invokeai/backend/minimax_h3/packing.py` * and `resolve_lowres_canvas_size` in `invokeai/backend/minimax_h3/presets.py`. + * - Ref2VA image references: `resolve_reference_image_short_edge` and + * `normalize_reference_image` in `invokeai/backend/minimax_h3/reference_conditioning.py`. * * Where the backend raises, these return null: the panel falls back to defaults * and reports the problem through `getVideoValidationReasons` instead of throwing. @@ -147,6 +154,63 @@ export const resolveMiniMaxH3Canvas = ( }; }; +/** Upstream's reference-image rule: a constant short edge, whatever the generation size is. */ +export const MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE = 2048; + +/** + * Pixels per packed row: H3 encodes at a 16x spatial compression and the transformer packs + * 2x2 latent patches, so a 32x32 pixel block is one row. + */ +export const MINIMAX_H3_ROW_PIXELS = 32 * 32; + +/** + * A reference image's normalized size and the rows it contributes. + * + * Those rows join the packed sequence and are re-attended at EVERY denoising step, with + * attention quadratic in the sequence length — which is the whole difference between the + * two detail settings. `'max'` pins the short edge to 2048 no matter how small the + * generation is; `'match'` scales the reference to the generation's pixel area (never + * above the 2048 rule), typically an order of magnitude fewer rows. + * + * Null when the inputs are degenerate, or when `'match'` has no target area to match — + * the panel then shows nothing rather than a wrong number. + */ +export const resolveMiniMaxH3ReferenceImage = ( + width: number, + height: number, + detail: VideoReferenceImageDetail, + targetArea: number | null +): { dimensions: VideoDimensions; rows: number } | null => { + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return null; + } + + let shortEdge = MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE; + + if (detail === 'match') { + if (targetArea === null || !Number.isFinite(targetArea) || targetArea <= 0) { + return null; + } + // The backend rounds with Python's banker's rounding here, so `roundHalfToEven` is + // what keeps this estimate equal to the size the graph actually encodes. + const matched = Math.max( + MINIMAX_H3_CANVAS_MULTIPLE, + roundHalfToEven(Math.min(width, height) * Math.sqrt(targetArea / (width * height))) + ); + + shortEdge = Math.min(MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE, matched); + } + + const scale = shortEdge / Math.min(width, height); + const dimensions = { + height: snapToMultiple(height * scale, MINIMAX_H3_CANVAS_MULTIPLE), + width: snapToMultiple(width * scale, MINIMAX_H3_CANVAS_MULTIPLE), + }; + + // Both axes are multiples of 32, so this is exact. + return { dimensions, rows: (dimensions.width * dimensions.height) / MINIMAX_H3_ROW_PIXELS }; +}; + /** The width/height parts of a preset ratio, for feeding the canvas resolvers. */ export const getVideoAspectRatioParts = (id: VideoAspectRatioId): VideoDimensions => { const [width = 1, height = 1] = id.split(':').map(Number); diff --git a/invokeai/frontend/webv2/src/features/video/core/settings.test.ts b/invokeai/frontend/webv2/src/features/video/core/settings.test.ts index 19f0ac57711..1173093c8b3 100644 --- a/invokeai/frontend/webv2/src/features/video/core/settings.test.ts +++ b/invokeai/frontend/webv2/src/features/video/core/settings.test.ts @@ -12,6 +12,7 @@ import { cloneVideoWidgetValues, createVideoSourceClip, deriveReferenceExtendClip, + getDefaultReferenceImageDetail, isVideoSettings, isVideoSourceClip, normalizeVideoSettings, @@ -260,6 +261,29 @@ describe('createVideoSourceClip', () => { }); }); +describe('getDefaultReferenceImageDetail', () => { + const imageReference = { + detail: 'max', + image: { height: 1080, image_name: 'ref.png', width: 1920 }, + kind: 'image', + } as const; + const videoReference = { + clip: SOURCE_VIDEO, + conditioning: 'video_audio', + kind: 'video', + } as const; + + it('starts the first image reference at maximum detail', () => { + expect(getDefaultReferenceImageDetail([])).toBe('max'); + expect(getDefaultReferenceImageDetail([videoReference])).toBe('max'); + }); + + it('matches the generation size once an image reference is placed', () => { + expect(getDefaultReferenceImageDetail([imageReference])).toBe('match'); + expect(getDefaultReferenceImageDetail([videoReference, imageReference])).toBe('match'); + }); +}); + describe('clearDeletedVideoMedia', () => { const withMedia = createSettings({ firstFrameImage: FIRST_FRAME, diff --git a/invokeai/frontend/webv2/src/features/video/core/settings.ts b/invokeai/frontend/webv2/src/features/video/core/settings.ts index 4cbc9cd8558..540c12958a4 100644 --- a/invokeai/frontend/webv2/src/features/video/core/settings.ts +++ b/invokeai/frontend/webv2/src/features/video/core/settings.ts @@ -18,6 +18,7 @@ import type { MiniMaxH3TargetResolution, VideoAspectRatioId, VideoGenerationMode, + VideoReferenceImageDetail, VideoReferenceItem, VideoSettings, VideoSourceClip, @@ -470,6 +471,22 @@ export const createVideoSourceClip = (item: { }; }; +/** + * The detail a newly added image reference starts on. + * + * The FIRST image reference keeps upstream's rule, a 2048px short edge: it is usually the + * subject the generation is about, and that is where the extra detail earns its cost. + * Later ones match the generation size instead. Reference rows are re-attended at every + * denoising step and attention is quadratic in the sequence, so a second and third 2048px + * reference are what turn a modest surcharge into a doubling — while the marginal value of + * conditioning a supporting reference at seven times the output's pixel density is small. + * + * Only the starting value: every card's selector still offers both, and the card shows the + * size and row count each choice produces. + */ +export const getDefaultReferenceImageDetail = (references: VideoReferenceItem[]): VideoReferenceImageDetail => + references.some((entry) => entry.kind === 'image') ? 'match' : 'max'; + /** The minimum frames a trim must keep — video_concat's crossfade consumes a 2-frame tail. */ export const MIN_VIDEO_TRIM_FRAMES = 2; diff --git a/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx b/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx index c990a04aa1b..da892de70d0 100644 --- a/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx +++ b/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx @@ -10,7 +10,8 @@ import { Badge, Box, createListCollection, HStack, Image, Input, Spinner, Stack, import { useDndContext, useDndMonitor, useDroppable } from '@dnd-kit/core'; import { galleryImages, galleryItems, galleryTransfers } from '@features/gallery'; import { galleryImageUrls, galleryVideoUrls, isGalleryItemDragData } from '@features/gallery/utility'; -import { createVideoSourceClip } from '@features/video/core/settings'; +import { resolveMiniMaxH3ReferenceImage } from '@features/video/core/dimensions'; +import { createVideoSourceClip, getDefaultReferenceImageDetail } from '@features/video/core/settings'; import { assertAccountScopeCurrent, captureAccountScope, @@ -74,6 +75,7 @@ const ReferenceCard = memo(function ReferenceCard({ onRemove, onUpdate, reference, + targetArea, }: { collections: ReferenceCollections; disabled: boolean; @@ -84,6 +86,8 @@ const ReferenceCard = memo(function ReferenceCard({ onRemove: (index: number) => void; onUpdate: (index: number, reference: VideoReferenceItem) => void; reference: VideoReferenceItem; + /** The generation's pixel area, which is what 'match' detail scales an image to. */ + targetArea: number | null; }) { const { t } = useTranslation(); const name = reference.kind === 'video' ? reference.clip.video_name : reference.image.image_name; @@ -128,6 +132,16 @@ const ReferenceCard = memo(function ReferenceCard({ }, [index, onUpdate, reference] ); + // What this reference will actually cost, at the size the graph will encode it: the two + // detail settings differ by an order of magnitude in rows, and nothing else in the panel + // says so before the generation is queued. + const imageCost = useMemo( + () => + reference.kind === 'image' + ? resolveMiniMaxH3ReferenceImage(reference.image.width, reference.image.height, reference.detail, targetArea) + : null, + [reference, targetArea] + ); const handleMoveUp = useCallback(() => onMove(index, -1), [index, onMove]); const handleMoveDown = useCallback(() => onMove(index, 1), [index, onMove]); const handleRemove = useCallback(() => onRemove(index), [index, onRemove]); @@ -160,6 +174,15 @@ const ReferenceCard = memo(function ReferenceCard({ value={selectValue} onValueChange={handleSelect} /> + {imageCost ? ( + + {t('widgets.video.referenceImageCost', { + height: imageCost.dimensions.height, + rows: imageCost.rows.toLocaleString(), + width: imageCost.dimensions.width, + })} + + ) : null} {/* One row per trim bound: the bound's live frame at left, its slider at right. The seeking thumbs replace the static gallery poster for video references — the start-frame thumb is the card's visual identity. */} @@ -246,6 +269,7 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({ maxVideos, onChange, references, + targetArea, }: { disabled?: boolean; maxImages: number; @@ -258,6 +282,8 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({ */ onChange: (update: (current: VideoReferenceItem[]) => VideoReferenceItem[]) => void; references: VideoReferenceItem[]; + /** The generation's pixel area, which sizes a 'match'-detail image reference. */ + targetArea: number | null; }) { const { t } = useTranslation(); const { getUploadBoardId, reportError, touchGalleryImages } = useVideoUiActions(); @@ -338,7 +364,10 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({ return [ ...current, { - detail: 'max', + // Read off the LIVE list, beside the cap re-check: which default applies + // depends on whether an image reference is already placed, and another + // writer can have placed one during the resolve above. + detail: getDefaultReferenceImageDetail(current), image: { height: resolved.height, image_name: resolved.imageName, width: resolved.width }, kind: 'image', }, @@ -533,6 +562,7 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({ canMoveDown={index < references.length - 1 && index + 1 !== anchorIndex} canMoveUp={index > 0 && index !== anchorIndex} reference={reference} + targetArea={targetArea} onMove={moveReference} onRemove={removeReference} onUpdate={updateReference} diff --git a/invokeai/frontend/webv2/src/features/video/ui/VideoWidgetView.tsx b/invokeai/frontend/webv2/src/features/video/ui/VideoWidgetView.tsx index ffa06b4db55..b6e401814c1 100644 --- a/invokeai/frontend/webv2/src/features/video/ui/VideoWidgetView.tsx +++ b/invokeai/frontend/webv2/src/features/video/ui/VideoWidgetView.tsx @@ -585,6 +585,7 @@ export const VideoWidgetView = () => { maxImages={policy.references?.maxImages ?? 9} maxVideos={policy.references?.maxVideos ?? 3} references={values.references} + targetArea={dimensions ? dimensions.width * dimensions.height : null} onChange={setReferences} />