diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 6681668ad15..f208c5e8787 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -3523,6 +3523,8 @@ "lastFrameExtendHelp": "Optional destination: the extension ends on this image.", "lastFrameHelp": "Optional: the video ends on this image (interpolates from the first frame).", "removeClip": "Remove video", + "sampleLength": "Sample Length (Frames)", + "sampleLengthWithSeconds": "Sample Length · {{seconds}}s", "trim": "Trim", "trimEnd": "End Frame", "trimEndShort": "End", diff --git a/invokeai/frontend/webv2/src/features/video/core/settings.test.ts b/invokeai/frontend/webv2/src/features/video/core/settings.test.ts index 19f0ac57711..ca57edd3943 100644 --- a/invokeai/frontend/webv2/src/features/video/core/settings.test.ts +++ b/invokeai/frontend/webv2/src/features/video/core/settings.test.ts @@ -4,6 +4,8 @@ import type { VideoReferenceItem, VideoSettings } from './types'; import { MINIMAX_H3_NUM_FRAMES_CHOICES } from './dimensions'; import { + resizeReferenceSampleWindow, + slideReferenceSampleWindow, applyReferenceExtendSourceVideo, applyReferenceExtendNumFrames, canPlaceReferenceExtendAnchor, @@ -869,3 +871,85 @@ describe('reference-extend linkage', () => { expect(applyReferenceExtendNumFrames(unlinked, 90)).toBe(unlinked); }); }); + +describe('reference sample window', () => { + const clip = (startFrame: number, endFrame: number, numFrames = 300) => ({ + endFrame, + fps: 24, + height: 480, + numFrames, + startFrame, + video_name: 'clip.mp4', + width: 640, + }); + + describe('slideReferenceSampleWindow', () => { + it('slides an ordinary window at constant length', () => { + const next = slideReferenceSampleWindow(clip(0, 199), 50, false); + expect([next.startFrame, next.endFrame]).toEqual([50, 249]); + }); + + it('stops at the clip end instead of shrinking (no overshoot ratchet)', () => { + // Drag far past the wall, then back to 0: the length must survive the round trip. + const overshot = slideReferenceSampleWindow(clip(0, 199), 299, false); + expect([overshot.startFrame, overshot.endFrame]).toEqual([100, 299]); + const back = slideReferenceSampleWindow(overshot, 0, false); + expect([back.startFrame, back.endFrame]).toEqual([0, 199]); + }); + + it('keeps the extend anchor end pinned to the cutpoint', () => { + // The anchor's seam continuity depends on frames adjacent to its end frame. + const next = slideReferenceSampleWindow(clip(180, 298), 200, true); + expect([next.startFrame, next.endFrame]).toEqual([200, 298]); + const backAndForth = slideReferenceSampleWindow(slideReferenceSampleWindow(next, 250, true), 200, true); + expect([backAndForth.startFrame, backAndForth.endFrame]).toEqual([200, 298]); + }); + + it('clamps the anchor start to its pinned end', () => { + const next = slideReferenceSampleWindow(clip(180, 298), 500, true); + expect([next.startFrame, next.endFrame]).toEqual([298, 298]); + }); + + it('self-heals a corrupt persisted trim into bounds', () => { + // end < start and end beyond the clip must both come back as a valid window. + const inverted = slideReferenceSampleWindow(clip(10, 5, 20), 0, false); + expect(inverted.startFrame).toBeGreaterThanOrEqual(0); + expect(inverted.endFrame).toBeGreaterThanOrEqual(inverted.startFrame); + expect(inverted.endFrame).toBeLessThanOrEqual(19); + const oversized = slideReferenceSampleWindow(clip(0, 999, 20), 5, false); + expect([oversized.startFrame, oversized.endFrame]).toEqual([0, 19]); + }); + + it('handles a single-frame clip', () => { + const next = slideReferenceSampleWindow(clip(0, 0, 1), 5, false); + expect([next.startFrame, next.endFrame]).toEqual([0, 0]); + }); + }); + + describe('resizeReferenceSampleWindow', () => { + it('grows an ordinary window forward from its start', () => { + const next = resizeReferenceSampleWindow(clip(50, 60), 100, false); + expect([next.startFrame, next.endFrame]).toEqual([50, 149]); + }); + + it('clamps the length to the clip end', () => { + const next = resizeReferenceSampleWindow(clip(250, 260), 100, false); + expect([next.startFrame, next.endFrame]).toEqual([250, 299]); + }); + + it('grows the extend anchor backward from its pinned end', () => { + const next = resizeReferenceSampleWindow(clip(280, 298), 100, true); + expect([next.startFrame, next.endFrame]).toEqual([199, 298]); + }); + + it('clamps the anchor lead-in at the clip start', () => { + const next = resizeReferenceSampleWindow(clip(280, 298), 1000, true); + expect([next.startFrame, next.endFrame]).toEqual([0, 298]); + }); + + it('never produces a window shorter than one frame', () => { + const next = resizeReferenceSampleWindow(clip(50, 199), -5, false); + expect([next.startFrame, next.endFrame]).toEqual([50, 50]); + }); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/video/core/settings.ts b/invokeai/frontend/webv2/src/features/video/core/settings.ts index 4cbc9cd8558..2445a82563a 100644 --- a/invokeai/frontend/webv2/src/features/video/core/settings.ts +++ b/invokeai/frontend/webv2/src/features/video/core/settings.ts @@ -79,6 +79,73 @@ export const isVideoSourceClip = (value: unknown): value is VideoSourceClip => export const VIDEO_REFERENCE_MAX_VIDEOS = 3; export const VIDEO_REFERENCE_MAX_IMAGES = 9; +/** + * Default sample length (in frames) for a newly added video reference. Reference rows cost + * VRAM in every denoise step — the packed sequence grows with reference length — so the + * useful sample is a short window that captures the wanted visual/audio features, not the + * whole clip. 200 frames ≈ 8s at the models' native 24 fps. + */ +export const DEFAULT_REFERENCE_SAMPLE_FRAMES = 200; + +/** + * Move a reference clip's sample window to a new start frame. + * + * Ordinary references slide at CONSTANT length, stopping at the clip's end rather than + * shrinking — a transient overshoot during a drag must not ratchet the sample down. The + * reference-extend anchor (`pinEnd`) instead keeps its end frame pinned to the Initial + * Video cutpoint (seam continuity depends on the frames adjacent to it; see + * deriveReferenceExtendClip), so moving its start only adjusts the lead-in length. + * + * Self-healing by construction: the returned window always satisfies + * 0 <= start <= end <= numFrames - 1, even from a corrupt persisted trim. + */ +export const slideReferenceSampleWindow = ( + clip: VideoSourceClip, + rawStart: number, + pinEnd: boolean +): VideoSourceClip => { + const maxFrame = Math.max(0, clip.numFrames - 1); + + if (pinEnd) { + const endFrame = Math.min(Math.max(0, clip.endFrame), maxFrame); + const startFrame = Math.min(Math.max(0, Math.round(rawStart)), endFrame); + + return { ...clip, endFrame, startFrame }; + } + + const sampleFrames = Math.min(Math.max(1, clip.endFrame - clip.startFrame + 1), maxFrame + 1); + const startFrame = Math.min(Math.max(0, Math.round(rawStart)), maxFrame - (sampleFrames - 1)); + + return { ...clip, endFrame: startFrame + sampleFrames - 1, startFrame }; +}; + +/** + * Resize a reference clip's sample window to a new length in frames. + * + * Ordinary references grow from the start frame (the end moves, clamped to the clip); the + * reference-extend anchor (`pinEnd`) grows backward from its pinned end (the start moves), + * since its end must stay on the Initial Video cutpoint. Same self-healing bounds as + * slideReferenceSampleWindow. + */ +export const resizeReferenceSampleWindow = ( + clip: VideoSourceClip, + rawSampleFrames: number, + pinEnd: boolean +): VideoSourceClip => { + const maxFrame = Math.max(0, clip.numFrames - 1); + const sampleFrames = Math.min(Math.max(1, Math.round(rawSampleFrames)), maxFrame + 1); + + if (pinEnd) { + const endFrame = Math.min(Math.max(0, clip.endFrame), maxFrame); + + return { ...clip, endFrame, startFrame: Math.max(0, endFrame - (sampleFrames - 1)) }; + } + + const startFrame = Math.min(Math.max(0, clip.startFrame), maxFrame); + + return { ...clip, endFrame: Math.min(startFrame + sampleFrames - 1, maxFrame), startFrame }; +}; + const VIDEO_REFERENCE_CONDITIONINGS = ['video_audio', 'video', 'audio'] as const; const VIDEO_REFERENCE_IMAGE_DETAILS = ['max', 'match'] as const; diff --git a/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx b/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx index 0cb7091a1a8..53ac4f20a8d 100644 --- a/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx +++ b/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx @@ -13,7 +13,12 @@ import { useDndContext, useDndMonitor, useDroppable } from '@dnd-kit/core'; import { galleryItems, galleryTransfers, toGalleryItemKey } from '@features/gallery'; import { GalleryPickerPopover } from '@features/gallery/picker'; import { galleryImageUrls, galleryVideoUrls, isGalleryItemDragData } from '@features/gallery/utility'; -import { createVideoSourceClip } from '@features/video/core/settings'; +import { + createVideoSourceClip, + DEFAULT_REFERENCE_SAMPLE_FRAMES, + resizeReferenceSampleWindow, + slideReferenceSampleWindow, +} from '@features/video/core/settings'; import { assertAccountScopeCurrent, captureAccountScope, @@ -22,7 +27,7 @@ import { import { Button, IconButton } from '@platform/ui/Button'; import { DropTargetOverlay } from '@platform/ui/DropTargetOverlay'; import { DropZone } from '@platform/ui/DropZone'; -import { Field } from '@platform/ui/Field'; +import { Field, FieldLabel } from '@platform/ui/Field'; import { MiddleTruncate } from '@platform/ui/MiddleTruncate'; import { Select } from '@platform/ui/Select'; import { SliderNumberField } from '@platform/ui/SliderNumberField'; @@ -109,28 +114,41 @@ const ReferenceCard = memo(function ReferenceCard({ }, [index, onUpdate, reference] ); + // The trim is presented as a sliding sample window — start frame plus length — because + // what the user is choosing is "how much" (every reference frame costs denoise VRAM) and + // "from where". Storage stays startFrame/endFrame (the request contract); the window + // math (constant-length slide that stops at the clip's end, and the extend anchor's + // pinned-to-the-cutpoint end) lives in core/settings. const handleStartFrame = useCallback( - (startFrame: number) => { + (rawStart: number) => { if (reference.kind === 'video') { onUpdate(index, { ...reference, - clip: { ...reference.clip, endFrame: Math.max(startFrame, reference.clip.endFrame), startFrame }, + clip: slideReferenceSampleWindow(reference.clip, rawStart, reference.fromSourceVideo === true), }); } }, [index, onUpdate, reference] ); - const handleEndFrame = useCallback( - (endFrame: number) => { + const handleSampleFrames = useCallback( + (rawSampleFrames: number) => { if (reference.kind === 'video') { onUpdate(index, { ...reference, - clip: { ...reference.clip, endFrame, startFrame: Math.min(reference.clip.startFrame, endFrame) }, + clip: resizeReferenceSampleWindow(reference.clip, rawSampleFrames, reference.fromSourceVideo === true), }); } }, [index, onUpdate, reference] ); + // The window's length, and the seconds it represents — the label carries the seconds + // because the control is how a user hits a target sample duration (reference frames cost + // denoise VRAM every step), while its unit has to stay frames to match the trim contract. + const sampleFrames = reference.kind === 'video' ? reference.clip.endFrame - reference.clip.startFrame + 1 : 0; + const sampleSeconds = + reference.kind === 'video' && Number.isFinite(reference.clip.fps) && reference.clip.fps > 0 + ? (sampleFrames / reference.clip.fps).toFixed(1) + : null; const handleMoveUp = useCallback(() => onMove(index, -1), [index, onMove]); const handleMoveDown = useCallback(() => onMove(index, 1), [index, onMove]); const handleRemove = useCallback(() => onRemove(index), [index, onRemove]); @@ -163,9 +181,12 @@ const ReferenceCard = memo(function ReferenceCard({ value={selectValue} onValueChange={handleSelect} /> - {/* One row per trim bound: the bound's live frame at left, its slider at - right. The seeking thumbs replace the static gallery poster for video - references — the start-frame thumb is the card's visual identity. */} + {/* One row per window edge: the live frame at left, its control at right. The + seeking thumbs replace the static gallery poster for video references — the + start-frame thumb is the card's visual identity. The second row's SLIDER is + the sample length (the quantity that costs VRAM); its THUMB still shows the + resulting end frame, badged with that frame number since the number field + beside it shows the length, not the frame. */} {reference.kind === 'video' ? ( @@ -175,36 +196,50 @@ const ReferenceCard = memo(function ReferenceCard({ label={t('widgets.video.trimStartShort')} src={galleryVideoUrls.full(name)} /> - + + {t('widgets.video.trimStart')} - + - + + + {sampleSeconds === null + ? t('widgets.video.sampleLength') + : t('widgets.video.sampleLengthWithSeconds', { seconds: sampleSeconds })} + - + ) : null} @@ -388,9 +423,15 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({ return [ ...current, { - // References are truncated to the generated duration, not joined: default to - // the whole clip rather than the extend-mode 2-frame-tail trim. - clip: { ...clip, endFrame: Math.max(0, clip.numFrames - 1), startFrame: 0 }, + // Default to a short sample window from the clip's start, not the whole + // clip: reference frames cost denoise VRAM every step, and a few seconds + // captures the wanted features. (Not the extend-mode 2-frame-tail trim + // either -- references are truncated to the generated duration, not joined.) + clip: { + ...clip, + endFrame: Math.max(0, Math.min(DEFAULT_REFERENCE_SAMPLE_FRAMES, clip.numFrames) - 1), + startFrame: 0, + }, conditioning: 'video_audio', kind: 'video', }, diff --git a/invokeai/frontend/webv2/src/features/video/ui/VideoSourceClipField.tsx b/invokeai/frontend/webv2/src/features/video/ui/VideoSourceClipField.tsx index 865d7efc78f..de197cd2813 100644 --- a/invokeai/frontend/webv2/src/features/video/ui/VideoSourceClipField.tsx +++ b/invokeai/frontend/webv2/src/features/video/ui/VideoSourceClipField.tsx @@ -324,6 +324,7 @@ export const VideoSourceClipField = memo( disabled={disabled} max={maxFrameIndex} min={0} + showStepper step={1} value={sourceVideo.startFrame} onChange={setStartFrame} @@ -343,6 +344,7 @@ export const VideoSourceClipField = memo( disabled={disabled} max={maxFrameIndex} min={0} + showStepper step={1} value={sourceVideo.endFrame} onChange={setEndFrame}