diff --git a/docs/src/content/docs/features/gallery.mdx b/docs/src/content/docs/features/gallery.mdx index 376f35d9d91..e4c0f97465e 100644 --- a/docs/src/content/docs/features/gallery.mdx +++ b/docs/src/content/docs/features/gallery.mdx @@ -149,8 +149,8 @@ Videos generated by InvokeAI (currently from the Wan 2.2 model family) appear al You can upload existing videos to a board via the standard drop-or-upload affordance. Everything is stored and served as **H.264/AAC MP4** so it plays in supported browsers, but the upload pipeline now converts on ingest rather than rejecting: - **H.264 in another container** (an iPhone `.mov` recorded with "Most Compatible", `.m4v`, most screen recordings) is losslessly remuxed — near-instant, no quality change. -- **Other codecs** (iPhone "High Efficiency" HEVC, ProRes, VP9, and anything else FFmpeg can decode) are transcoded to H.264. Expect the upload to take roughly the clip's duration on a typical CPU. 10-bit HDR footage is converted to 8-bit without tonemapping, so HDR clips will look flatter than the original — pre-convert those yourself if color fidelity matters. -- **Audio files** (`.mp3`, `.m4a`, `.wav`, `.flac`, `.ogg`, and friends) are wrapped into a video whose frames render the waveform. The clip behaves like any other gallery video — it can be trimmed, played, and used wherever audio-only conditioning accepts a video — without audio being a separate media type. +- **Other codecs** (iPhone "High Efficiency" HEVC, ProRes, VP9, Windows Media `.wmv`, and anything else FFmpeg can decode) are transcoded to H.264. Expect the upload to take roughly the clip's duration on a typical CPU. 10-bit HDR footage is converted to 8-bit without tonemapping, so HDR clips will look flatter than the original — pre-convert those yourself if color fidelity matters. +- **Audio files** (`.mp3`, `.m4a`, `.wav`, `.flac`, `.ogg`, `.wma`, and friends) are wrapped into a video whose frames render the waveform. The clip behaves like any other gallery video — it can be trimmed, played, and used wherever audio-only conditioning accepts a video — without audio being a separate media type. The **Concatenate Videos** and **Extract Video Range** nodes encode video frames only; audio tracks are not preserved. Concatenation requires matching source frame rates unless an explicit output FPS is selected, which retimes the source frames. diff --git a/invokeai/app/api/routers/videos.py b/invokeai/app/api/routers/videos.py index 65f5c03b234..9f6821e057c 100644 --- a/invokeai/app/api/routers/videos.py +++ b/invokeai/app/api/routers/videos.py @@ -70,9 +70,37 @@ # are wrapped into waveform videos so audio clips flow through the video pipeline # (gallery, trim, audio-only reference conditioning) without a first-class audio type. ACCEPTED_VIDEO_MIME_PREFIXES = ("video/",) -ACCEPTED_VIDEO_EXTENSIONS = (".mp4", ".mov", ".m4v", ".webm", ".mkv", ".avi", ".mpg", ".mpeg", ".3gp") +# The extension lists are the fallback for uploads whose type the browser could not +# determine (they arrive as application/octet-stream); the MIME prefixes above accept the +# ordinary case. Everything here is demuxable and decodable by the bundled ffmpeg — ASF +# (Windows Media) included — and lands as H.264/AAC MP4 through the ingest converter. +ACCEPTED_VIDEO_EXTENSIONS = ( + ".mp4", + ".mov", + ".m4v", + ".webm", + ".mkv", + ".avi", + ".mpg", + ".mpeg", + ".3gp", + ".wmv", + ".asf", +) ACCEPTED_AUDIO_MIME_PREFIXES = ("audio/",) -ACCEPTED_AUDIO_EXTENSIONS = (".mp3", ".m4a", ".aac", ".wav", ".flac", ".ogg", ".oga", ".opus", ".aiff", ".aif") +ACCEPTED_AUDIO_EXTENSIONS = ( + ".mp3", + ".m4a", + ".aac", + ".wav", + ".flac", + ".ogg", + ".oga", + ".opus", + ".aiff", + ".aif", + ".wma", +) # Per-chunk size for HTTP Range responses (1 MB) RANGE_CHUNK_SIZE = 1024 * 1024 diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index b8b7b87a6f2..9faafcbd3a8 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -3572,13 +3572,13 @@ "referenceExtendHelp": "The generated video is appended to this clip at its End Frame. A linked video reference samples up to the last ~5 seconds before that cutpoint for continuity — never more than the generated clip's own length, since the model discards the overrun at the seam. Its trim re-derives whenever the cutpoints or the frame count change.", "referenceImageCapRace": "All {{max}} image reference slots filled while this one was loading, so it was not added.", "referenceImageCost": "{{width}}×{{height}} · {{rows}} rows per step", - "referenceVideoCapRace": "All {{max}} video reference slots filled while this one was loading, so it was not added.", + "referenceVideoCapRace": "All {{max}} video or audio reference slots filled while this one was loading, so it was not added.", "referenceFromInitialVideo": "Initial video", "references": "References", - "referencesHelp": "Up to 3 videos and 9 images condition the generation, in order — reordering references changes the result. A video reference can contribute its image track, its soundtrack, or both. Every reference adds rows to each denoising step, so the first image reference starts at maximum detail and later ones match the generation size.", + "referencesHelp": "Up to 3 videos and 9 images condition the generation, in order — reordering references changes the result. A video reference can contribute its image track, its soundtrack, or both. An uploaded audio file becomes a waveform clip that fills a video slot and conditions on its soundtrack. Every reference adds rows to each denoising step, so the first image reference starts at maximum detail and later ones match the generation size.", "removeReference": "Remove reference", "staleReferences": "References are set but the selected model cannot use them.", - "uploadVideoReference": "Upload video" + "uploadVideoReference": "Upload video or audio" } }, "workflowLibrary": { diff --git a/invokeai/frontend/webv2/src/features/gallery/data/backend.test.ts b/invokeai/frontend/webv2/src/features/gallery/data/backend.test.ts index 6191e06b077..66f8cea2010 100644 --- a/invokeai/frontend/webv2/src/features/gallery/data/backend.test.ts +++ b/invokeai/frontend/webv2/src/features/gallery/data/backend.test.ts @@ -782,6 +782,8 @@ describe('classifyGalleryUpload', () => { ['application/octet-stream', 'photo.jpeg', 'image'], ['application/octet-stream', 'song.mp3', 'video'], ['binary/octet-stream', 'clip.MP4', 'video'], + ['application/octet-stream', 'clip.wmv', 'video'], + ['application/octet-stream', 'song.WMA', 'video'], ['application/pdf', 'photo.png', 'image'], ] as const)('classifies MIME %s and name %s as %s', (type, name, kind) => { expect(classifyGalleryUpload(new File(['media'], name, { type }))).toEqual({ kind }); diff --git a/invokeai/frontend/webv2/src/features/gallery/data/backend.ts b/invokeai/frontend/webv2/src/features/gallery/data/backend.ts index 2c3baf41996..02b1942f447 100644 --- a/invokeai/frontend/webv2/src/features/gallery/data/backend.ts +++ b/invokeai/frontend/webv2/src/features/gallery/data/backend.ts @@ -102,6 +102,8 @@ const GALLERY_UPLOAD_KIND_BY_EXTENSION = new Map([ ['.mpg', 'video'], ['.mpeg', 'video'], ['.3gp', 'video'], + ['.wmv', 'video'], + ['.asf', 'video'], ['.mp3', 'video'], ['.m4a', 'video'], ['.aac', 'video'], @@ -112,6 +114,7 @@ const GALLERY_UPLOAD_KIND_BY_EXTENSION = new Map([ ['.opus', 'video'], ['.aiff', 'video'], ['.aif', 'video'], + ['.wma', 'video'], ]); export const classifyGalleryUpload = (file: Pick): { kind: GalleryUploadKind } | null => { diff --git a/invokeai/frontend/webv2/src/features/video/core/settings.test.ts b/invokeai/frontend/webv2/src/features/video/core/settings.test.ts index 8225441a907..70b35d91946 100644 --- a/invokeai/frontend/webv2/src/features/video/core/settings.test.ts +++ b/invokeai/frontend/webv2/src/features/video/core/settings.test.ts @@ -14,6 +14,7 @@ import { cloneVideoWidgetValues, createVideoSourceClip, deriveReferenceExtendClip, + getDefaultReferenceConditioning, getDefaultReferenceImageDetail, isVideoSettings, isVideoSourceClip, @@ -263,6 +264,23 @@ describe('createVideoSourceClip', () => { }); }); +describe('getDefaultReferenceConditioning', () => { + it('starts a wrapped audio upload on its soundtrack alone', () => { + expect(getDefaultReferenceConditioning({ media_origin: 'audio_upload' })).toBe('audio'); + }); + + it('keeps video + audio for ordinary videos', () => { + expect(getDefaultReferenceConditioning({ generation_mode: 'minimax_h3_ref2v' })).toBe('video_audio'); + expect(getDefaultReferenceConditioning({ media_origin: 'something_else' })).toBe('video_audio'); + }); + + it('keeps video + audio when there is no metadata to read', () => { + expect(getDefaultReferenceConditioning(null)).toBe('video_audio'); + expect(getDefaultReferenceConditioning(undefined)).toBe('video_audio'); + expect(getDefaultReferenceConditioning({})).toBe('video_audio'); + }); +}); + describe('getDefaultReferenceImageDetail', () => { const imageReference = { detail: 'max', diff --git a/invokeai/frontend/webv2/src/features/video/core/settings.ts b/invokeai/frontend/webv2/src/features/video/core/settings.ts index 97ad1ebdec1..67ff8bd2254 100644 --- a/invokeai/frontend/webv2/src/features/video/core/settings.ts +++ b/invokeai/frontend/webv2/src/features/video/core/settings.ts @@ -18,6 +18,7 @@ import type { MiniMaxH3TargetResolution, VideoAspectRatioId, VideoGenerationMode, + VideoReferenceConditioning, VideoReferenceImageDetail, VideoReferenceItem, VideoSettings, @@ -538,6 +539,20 @@ export const createVideoSourceClip = (item: { }; }; +/** + * The conditioning a video reference starts on when it is added from the gallery or an + * upload. + * + * Audio uploads are stored as videos: the server wraps an uploaded audio file into a + * rendered-waveform clip at ingest and stamps `media_origin: audio_upload` on it. Those + * frames are a picture of the sound rather than footage anyone means to condition on, so + * such a reference defaults to its soundtrack alone. Everything else keeps video + audio. + * This is only the starting value — the card's selector still offers all three. + */ +export const getDefaultReferenceConditioning = ( + metadata: Record | null | undefined +): VideoReferenceConditioning => (metadata?.media_origin === 'audio_upload' ? 'audio' : 'video_audio'); + /** * The detail a newly added image reference starts on. * diff --git a/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx b/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx index ce54bbb59f1..3f0f03b12a1 100644 --- a/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx +++ b/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx @@ -10,13 +10,14 @@ import type { ChangeEvent } from 'react'; import { Badge, Box, createListCollection, HStack, Icon, Image, Input, Spinner, Stack, Text } from '@chakra-ui/react'; import { useDndContext, useDndMonitor, useDroppable } from '@dnd-kit/core'; -import { galleryItems, galleryTransfers, toGalleryItemKey } from '@features/gallery'; +import { galleryItems, galleryTransfers, galleryVideos, toGalleryItemKey } from '@features/gallery'; import { GalleryPickerPopover } from '@features/gallery/picker'; import { galleryImageUrls, galleryVideoUrls, isGalleryItemDragData } from '@features/gallery/utility'; import { resolveMiniMaxH3ReferenceImage } from '@features/video/core/dimensions'; import { createVideoSourceClip, DEFAULT_REFERENCE_SAMPLE_FRAMES, + getDefaultReferenceConditioning, getDefaultReferenceImageDetail, resizeReferenceSampleWindow, slideReferenceSampleWindow, @@ -49,6 +50,37 @@ import { useVideoUiActions } from './VideoUiContext'; const DROP_ID = 'video-reference-list'; const IMAGE_UPLOAD_ACCEPT = 'image/png,image/jpeg,image/webp,.png,.jpg,.jpeg,.webp'; +// One upload button for both media kinds: an uploaded audio file becomes a waveform video, +// so it occupies a VIDEO reference slot and shares that cap -- a separate audio button would +// grey out with this one. The wildcards cover the ordinary case; the explicit extensions +// (mirroring the upload route's accepted lists) are what match a file whose type the OS +// could not map, which the browser then offers as octet-stream. +const MEDIA_UPLOAD_ACCEPT = [ + 'video/*', + 'audio/*', + '.mp4', + '.mov', + '.m4v', + '.webm', + '.mkv', + '.avi', + '.mpg', + '.mpeg', + '.3gp', + '.wmv', + '.asf', + '.mp3', + '.m4a', + '.aac', + '.wav', + '.flac', + '.ogg', + '.oga', + '.opus', + '.aiff', + '.aif', + '.wma', +].join(','); const DROP_ZONE_FOCUS_PROPS = { outlineColor: 'accent.focusRing', outlineOffset: '2px', @@ -436,40 +468,47 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({ ); const addVideoItem = useCallback( - (item: GalleryVideoItem) => { + // The conditioning is passed in rather than derived here: only a caller holding the + // clip's metadata can tell a wrapped audio upload from footage. Callers without it get + // the ordinary video default -- and the entry back, so a late answer can correct it. + (item: GalleryVideoItem, conditioning: VideoReferenceConditioning = 'video_audio') => { const clip = createVideoSourceClip(item); + // Built outside the updater so the caller holds the same object the list does: it is + // the only durable handle on this entry once reordering moves it. + const entry: Extract = { + // Default to a short sample window from the clip's start, not the whole + // clip: reference frames cost denoise VRAM every step, and a few seconds + // captures the wanted features. (Not the extend-mode 2-frame-tail trim + // either -- references are truncated to the generated duration, not joined.) + clip: { + ...clip, + endFrame: Math.max(0, Math.min(DEFAULT_REFERENCE_SAMPLE_FRAMES, clip.numFrames) - 1), + startFrame: 0, + }, + conditioning, + kind: 'video', + }; // Same live cap re-check as the image path -- the Initial Video's // anchor is the writer that most easily fills the slots mid-await. let declined = false; setErrorMessage(null); onChange((current) => { - if (current.filter((entry) => entry.kind === 'video').length >= maxVideos) { + if (current.filter((existing) => existing.kind === 'video').length >= maxVideos) { declined = true; return current; } - return [ - ...current, - { - // Default to a short sample window from the clip's start, not the whole - // clip: reference frames cost denoise VRAM every step, and a few seconds - // captures the wanted features. (Not the extend-mode 2-frame-tail trim - // either -- references are truncated to the generated duration, not joined.) - clip: { - ...clip, - endFrame: Math.max(0, Math.min(DEFAULT_REFERENCE_SAMPLE_FRAMES, clip.numFrames) - 1), - startFrame: 0, - }, - conditioning: 'video_audio', - kind: 'video', - }, - ]; + return [...current, entry]; }); if (declined) { setErrorMessage(t('widgets.video.referenceVideoCapRace', { max: maxVideos })); + + return null; } + + return entry; }, [maxVideos, onChange, t] ); @@ -480,10 +519,17 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({ setIsLoading(true); try { - const item = await galleryItems.resolve({ kind: 'video', name: videoName }); + // Fetched alongside the resolve, not after it: the metadata only picks the + // card's starting conditioning, and it must not add a round trip to the add. + // A missing or unreadable record is not a failure -- it just means the + // ordinary video default. + const [item, metadata] = await Promise.all([ + galleryItems.resolve({ kind: 'video', name: videoName }), + galleryVideos.metadata(videoName).catch(() => null), + ]); if (item?.kind === 'video') { - addVideoItem(item); + addVideoItem(item, getDefaultReferenceConditioning(metadata)); } } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -585,15 +631,49 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({ }), [imageCount, maxImages, maxVideos, references, videoCount] ); + const addPickedVideo = useCallback( + (item: GalleryVideoItem) => { + // The card goes in SYNCHRONOUSLY and is corrected afterwards, rather than waiting on + // the metadata the way the drop and upload paths do. The picker stays open and judges + // each click against the reference list as it stands -- an add that had not landed yet + // would leave the tile pickable (a second click would duplicate it), leave the + // remaining count stale, and let two picks land in whichever order their fetches + // finished, which for references is a different generation. + const entry = addVideoItem(item); + + if (!entry) { + return; + } + + // The metadata is the only thing that tells a wrapped audio upload from footage. An + // unreadable record is not a failure -- the ordinary video default just stands. + void galleryVideos + .metadata(item.name) + .then((metadata) => { + const conditioning = getDefaultReferenceConditioning(metadata); + + if (conditioning === entry.conditioning) { + return; + } + // Matched by identity, not index: a card the user has since edited is a different + // object and keeps their choice, and a removed one is simply no longer there. + onChange((current) => + current.map((existing) => (existing === entry ? { ...entry, conditioning } : existing)) + ); + }) + .catch(() => undefined); + }, + [addVideoItem, onChange] + ); const handlePick = useCallback( (item: GalleryItem) => { if (item.kind === 'video') { - addVideoItem(item); + addPickedVideo(item); } else { addImageReference(item); } }, - [addImageReference, addVideoItem] + [addImageReference, addPickedVideo] ); const updateReference = useCallback( @@ -693,7 +773,7 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({ {/* Audio files upload too: the server wraps them into waveform videos, which is how audio-only reference clips enter the pipeline. */} - + ); }); diff --git a/tests/app/routers/test_videos_multiuser.py b/tests/app/routers/test_videos_multiuser.py index 984c8dffb6a..25c3b9a3700 100644 --- a/tests/app/routers/test_videos_multiuser.py +++ b/tests/app/routers/test_videos_multiuser.py @@ -12,20 +12,23 @@ """ import inspect +import io import json from pathlib import Path from typing import Any from unittest.mock import MagicMock, patch import pytest -from fastapi import status +from fastapi import UploadFile, status from fastapi.testclient import TestClient from pydantic import ValidationError +from starlette.datastructures import Headers from invokeai.app.api.dependencies import ApiDependencies from invokeai.app.api.routers import videos as videos_router_module from invokeai.app.api.routers.videos import ( VideoNamesBatch, + _classify_upload, _is_mp4_file, delete_uncategorized_videos, delete_video, @@ -788,6 +791,29 @@ def test_upload_rejects_unrecognized_file_kind(client: TestClient, mock_invoker: assert response.status_code == status.HTTP_415_UNSUPPORTED_MEDIA_TYPE +@pytest.mark.parametrize( + ("filename", "expected"), + [ + ("clip.wmv", "video"), + ("clip.asf", "video"), + ("song.wma", "audio"), + ("clip.MOV", "video"), + ("voice.M4A", "audio"), + ("notes.txt", None), + ], +) +def test_classify_upload_falls_back_to_the_extension(filename: str, expected: str | None) -> None: + """A file whose type the browser could not map arrives as octet-stream, leaving the + extension as the only signal. The picker offers these, so the route must accept them.""" + upload = UploadFile( + file=io.BytesIO(b""), + filename=filename, + headers=Headers({"content-type": "application/octet-stream"}), + ) + + assert _classify_upload(upload) == expected + + def test_mp4_validation_allows_boxes_before_file_type(tmp_path: Path) -> None: path = tmp_path / "valid.mp4" path.write_bytes(b"\x00\x00\x00\x08free" + b"\x00\x00\x00\x18ftypmp42" + b"\x00" * 12)