invoke-ai · lstein · Sep 7, 2026 · Sep 3, 2026 · Sep 3, 2026 · Sep 3, 2026
diff --git a/docs/src/content/docs/features/gallery.mdx b/docs/src/content/docs/features/gallery.mdx
@@ -149,8 +149,8 @@ Videos generated by InvokeAI (currently from the Wan 2.2 model family) appear al
 You can upload existing videos to a board via the standard drop-or-upload affordance. Everything is stored and served as **H.264/AAC MP4** so it plays in supported browsers, but the upload pipeline now converts on ingest rather than rejecting:
 
 - **H.264 in another container** (an iPhone `.mov` recorded with "Most Compatible", `.m4v`, most screen recordings) is losslessly remuxed — near-instant, no quality change.
-- **Other codecs** (iPhone "High Efficiency" HEVC, ProRes, VP9, and anything else FFmpeg can decode) are transcoded to H.264. Expect the upload to take roughly the clip's duration on a typical CPU. 10-bit HDR footage is converted to 8-bit without tonemapping, so HDR clips will look flatter than the original — pre-convert those yourself if color fidelity matters.
-- **Audio files** (`.mp3`, `.m4a`, `.wav`, `.flac`, `.ogg`, and friends) are wrapped into a video whose frames render the waveform. The clip behaves like any other gallery video — it can be trimmed, played, and used wherever audio-only conditioning accepts a video — without audio being a separate media type.
+- **Other codecs** (iPhone "High Efficiency" HEVC, ProRes, VP9, Windows Media `.wmv`, and anything else FFmpeg can decode) are transcoded to H.264. Expect the upload to take roughly the clip's duration on a typical CPU. 10-bit HDR footage is converted to 8-bit without tonemapping, so HDR clips will look flatter than the original — pre-convert those yourself if color fidelity matters.
+- **Audio files** (`.mp3`, `.m4a`, `.wav`, `.flac`, `.ogg`, `.wma`, and friends) are wrapped into a video whose frames render the waveform. The clip behaves like any other gallery video — it can be trimmed, played, and used wherever audio-only conditioning accepts a video — without audio being a separate media type.
 
 The **Concatenate Videos** and **Extract Video Range** nodes encode video frames only; audio tracks are not preserved. Concatenation requires matching source frame rates unless an explicit output FPS is selected, which retimes the source frames.
 

@@ -70,9 +70,37 @@
 # are wrapped into waveform videos so audio clips flow through the video pipeline
 # (gallery, trim, audio-only reference conditioning) without a first-class audio type.
 ACCEPTED_VIDEO_MIME_PREFIXES = ("video/",)
-ACCEPTED_VIDEO_EXTENSIONS = (".mp4", ".mov", ".m4v", ".webm", ".mkv", ".avi", ".mpg", ".mpeg", ".3gp")
+# The extension lists are the fallback for uploads whose type the browser could not
+# determine (they arrive as application/octet-stream); the MIME prefixes above accept the
+# ordinary case. Everything here is demuxable and decodable by the bundled ffmpeg — ASF
+# (Windows Media) included — and lands as H.264/AAC MP4 through the ingest converter.
+ACCEPTED_VIDEO_EXTENSIONS = (
+    ".mp4",
+    ".mov",
+    ".m4v",
+    ".webm",
+    ".mkv",
+    ".avi",
+    ".mpg",
+    ".mpeg",
+    ".3gp",
+    ".wmv",
+    ".asf",
+)
 ACCEPTED_AUDIO_MIME_PREFIXES = ("audio/",)
-ACCEPTED_AUDIO_EXTENSIONS = (".mp3", ".m4a", ".aac", ".wav", ".flac", ".ogg", ".oga", ".opus", ".aiff", ".aif")
+ACCEPTED_AUDIO_EXTENSIONS = (
+    ".mp3",
+    ".m4a",
+    ".aac",
+    ".wav",
+    ".flac",
+    ".ogg",
+    ".oga",
+    ".opus",
+    ".aiff",
+    ".aif",
+    ".wma",
+)
 
 # Per-chunk size for HTTP Range responses (1 MB)
 RANGE_CHUNK_SIZE = 1024 * 1024

@@ -3572,13 +3572,13 @@
       "referenceExtendHelp": "The generated video is appended to this clip at its End Frame. A linked video reference samples up to the last ~5 seconds before that cutpoint for continuity — never more than the generated clip's own length, since the model discards the overrun at the seam. Its trim re-derives whenever the cutpoints or the frame count change.",
       "referenceImageCapRace": "All {{max}} image reference slots filled while this one was loading, so it was not added.",
       "referenceImageCost": "{{width}}×{{height}} · {{rows}} rows per step",
-      "referenceVideoCapRace": "All {{max}} video reference slots filled while this one was loading, so it was not added.",
+      "referenceVideoCapRace": "All {{max}} video or audio reference slots filled while this one was loading, so it was not added.",
       "referenceFromInitialVideo": "Initial video",
       "references": "References",
-      "referencesHelp": "Up to 3 videos and 9 images condition the generation, in order — reordering references changes the result. A video reference can contribute its image track, its soundtrack, or both. Every reference adds rows to each denoising step, so the first image reference starts at maximum detail and later ones match the generation size.",
+      "referencesHelp": "Up to 3 videos and 9 images condition the generation, in order — reordering references changes the result. A video reference can contribute its image track, its soundtrack, or both. An uploaded audio file becomes a waveform clip that fills a video slot and conditions on its soundtrack. Every reference adds rows to each denoising step, so the first image reference starts at maximum detail and later ones match the generation size.",
       "removeReference": "Remove reference",
       "staleReferences": "References are set but the selected model cannot use them.",
-      "uploadVideoReference": "Upload video"
+      "uploadVideoReference": "Upload video or audio"
     }
   },
   "workflowLibrary": {

@@ -782,6 +782,8 @@ describe('classifyGalleryUpload', () => {
     ['application/octet-stream', 'photo.jpeg', 'image'],
     ['application/octet-stream', 'song.mp3', 'video'],
     ['binary/octet-stream', 'clip.MP4', 'video'],
+    ['application/octet-stream', 'clip.wmv', 'video'],
+    ['application/octet-stream', 'song.WMA', 'video'],
     ['application/pdf', 'photo.png', 'image'],
   ] as const)('classifies MIME %s and name %s as %s', (type, name, kind) => {
     expect(classifyGalleryUpload(new File(['media'], name, { type }))).toEqual({ kind });

@@ -102,6 +102,8 @@ const GALLERY_UPLOAD_KIND_BY_EXTENSION = new Map<string, GalleryUploadKind>([
   ['.mpg', 'video'],
   ['.mpeg', 'video'],
   ['.3gp', 'video'],
+  ['.wmv', 'video'],
+  ['.asf', 'video'],
   ['.mp3', 'video'],
   ['.m4a', 'video'],
   ['.aac', 'video'],
@@ -112,6 +114,7 @@ const GALLERY_UPLOAD_KIND_BY_EXTENSION = new Map<string, GalleryUploadKind>([
   ['.opus', 'video'],
   ['.aiff', 'video'],
   ['.aif', 'video'],
+  ['.wma', 'video'],
 ]);
 
 export const classifyGalleryUpload = (file: Pick<File, 'name' | 'type'>): { kind: GalleryUploadKind } | null => {

@@ -14,6 +14,7 @@ import {
   cloneVideoWidgetValues,
   createVideoSourceClip,
   deriveReferenceExtendClip,
+  getDefaultReferenceConditioning,
   getDefaultReferenceImageDetail,
   isVideoSettings,
   isVideoSourceClip,
@@ -263,6 +264,23 @@ describe('createVideoSourceClip', () => {
   });
 });
 
+describe('getDefaultReferenceConditioning', () => {
+  it('starts a wrapped audio upload on its soundtrack alone', () => {
+    expect(getDefaultReferenceConditioning({ media_origin: 'audio_upload' })).toBe('audio');
+  });
+
+  it('keeps video + audio for ordinary videos', () => {
+    expect(getDefaultReferenceConditioning({ generation_mode: 'minimax_h3_ref2v' })).toBe('video_audio');
+    expect(getDefaultReferenceConditioning({ media_origin: 'something_else' })).toBe('video_audio');
+  });
+
+  it('keeps video + audio when there is no metadata to read', () => {
+    expect(getDefaultReferenceConditioning(null)).toBe('video_audio');
+    expect(getDefaultReferenceConditioning(undefined)).toBe('video_audio');
+    expect(getDefaultReferenceConditioning({})).toBe('video_audio');
+  });
+});
+
 describe('getDefaultReferenceImageDetail', () => {
   const imageReference = {
     detail: 'max',

@@ -18,6 +18,7 @@ import type {
   MiniMaxH3TargetResolution,
   VideoAspectRatioId,
   VideoGenerationMode,
+  VideoReferenceConditioning,
   VideoReferenceImageDetail,
   VideoReferenceItem,
   VideoSettings,
@@ -538,6 +539,20 @@ export const createVideoSourceClip = (item: {
   };
 };
 
+/**
+ * The conditioning a video reference starts on when it is added from the gallery or an
+ * upload.
+ *
+ * Audio uploads are stored as videos: the server wraps an uploaded audio file into a
+ * rendered-waveform clip at ingest and stamps `media_origin: audio_upload` on it. Those
+ * frames are a picture of the sound rather than footage anyone means to condition on, so
+ * such a reference defaults to its soundtrack alone. Everything else keeps video + audio.
+ * This is only the starting value — the card's selector still offers all three.
+ */
+export const getDefaultReferenceConditioning = (
+  metadata: Record<string, unknown> | null | undefined
+): VideoReferenceConditioning => (metadata?.media_origin === 'audio_upload' ? 'audio' : 'video_audio');
+
 /**
  * The detail a newly added image reference starts on.
  *

@@ -10,13 +10,14 @@ import type { ChangeEvent } from 'react';
 
 import { Badge, Box, createListCollection, HStack, Icon, Image, Input, Spinner, Stack, Text } from '@chakra-ui/react';
 import { useDndContext, useDndMonitor, useDroppable } from '@dnd-kit/core';
-import { galleryItems, galleryTransfers, toGalleryItemKey } from '@features/gallery';
+import { galleryItems, galleryTransfers, galleryVideos, toGalleryItemKey } from '@features/gallery';
 import { GalleryPickerPopover } from '@features/gallery/picker';
 import { galleryImageUrls, galleryVideoUrls, isGalleryItemDragData } from '@features/gallery/utility';
 import { resolveMiniMaxH3ReferenceImage } from '@features/video/core/dimensions';
 import {
   createVideoSourceClip,
   DEFAULT_REFERENCE_SAMPLE_FRAMES,
+  getDefaultReferenceConditioning,
   getDefaultReferenceImageDetail,
   resizeReferenceSampleWindow,
   slideReferenceSampleWindow,
@@ -49,6 +50,37 @@ import { useVideoUiActions } from './VideoUiContext';
 
 const DROP_ID = 'video-reference-list';
 const IMAGE_UPLOAD_ACCEPT = 'image/png,image/jpeg,image/webp,.png,.jpg,.jpeg,.webp';
+// One upload button for both media kinds: an uploaded audio file becomes a waveform video,
+// so it occupies a VIDEO reference slot and shares that cap -- a separate audio button would
+// grey out with this one. The wildcards cover the ordinary case; the explicit extensions
+// (mirroring the upload route's accepted lists) are what match a file whose type the OS
+// could not map, which the browser then offers as octet-stream.
+const MEDIA_UPLOAD_ACCEPT = [
+  'video/*',
+  'audio/*',
+  '.mp4',
+  '.mov',
+  '.m4v',
+  '.webm',
+  '.mkv',
+  '.avi',
+  '.mpg',
+  '.mpeg',
+  '.3gp',
+  '.wmv',
+  '.asf',
+  '.mp3',
+  '.m4a',
+  '.aac',
+  '.wav',
+  '.flac',
+  '.ogg',
+  '.oga',
+  '.opus',
+  '.aiff',
+  '.aif',
+  '.wma',
+].join(',');
 const DROP_ZONE_FOCUS_PROPS = {
   outlineColor: 'accent.focusRing',
   outlineOffset: '2px',
@@ -436,40 +468,47 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({
   );
 
   const addVideoItem = useCallback(
-    (item: GalleryVideoItem) => {
+    // The conditioning is passed in rather than derived here: only a caller holding the
+    // clip's metadata can tell a wrapped audio upload from footage. Callers without it get
+    // the ordinary video default -- and the entry back, so a late answer can correct it.
+    (item: GalleryVideoItem, conditioning: VideoReferenceConditioning = 'video_audio') => {
       const clip = createVideoSourceClip(item);
+      // Built outside the updater so the caller holds the same object the list does: it is
+      // the only durable handle on this entry once reordering moves it.
+      const entry: Extract<VideoReferenceItem, { kind: 'video' }> = {
+        // Default to a short sample window from the clip's start, not the whole
+        // clip: reference frames cost denoise VRAM every step, and a few seconds
+        // captures the wanted features. (Not the extend-mode 2-frame-tail trim
+        // either -- references are truncated to the generated duration, not joined.)
+        clip: {
+          ...clip,
+          endFrame: Math.max(0, Math.min(DEFAULT_REFERENCE_SAMPLE_FRAMES, clip.numFrames) - 1),
+          startFrame: 0,
+        },
+        conditioning,
+        kind: 'video',
+      };
       // Same live cap re-check as the image path -- the Initial Video's
       // anchor is the writer that most easily fills the slots mid-await.
       let declined = false;
 
       setErrorMessage(null);
       onChange((current) => {
-        if (current.filter((entry) => entry.kind === 'video').length >= maxVideos) {
+        if (current.filter((existing) => existing.kind === 'video').length >= maxVideos) {
           declined = true;
 
           return current;
         }
 
-        return [
-          ...current,
-          {
-            // Default to a short sample window from the clip's start, not the whole
-            // clip: reference frames cost denoise VRAM every step, and a few seconds
-            // captures the wanted features. (Not the extend-mode 2-frame-tail trim
-            // either -- references are truncated to the generated duration, not joined.)
-            clip: {
-              ...clip,
-              endFrame: Math.max(0, Math.min(DEFAULT_REFERENCE_SAMPLE_FRAMES, clip.numFrames) - 1),
-              startFrame: 0,
-            },
-            conditioning: 'video_audio',
-            kind: 'video',
-          },
-        ];
+        return [...current, entry];
       });
       if (declined) {
         setErrorMessage(t('widgets.video.referenceVideoCapRace', { max: maxVideos }));
+
+        return null;
       }
+
+      return entry;
     },
     [maxVideos, onChange, t]
   );
@@ -480,10 +519,17 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({
       setIsLoading(true);
 
       try {
-        const item = await galleryItems.resolve({ kind: 'video', name: videoName });
+        // Fetched alongside the resolve, not after it: the metadata only picks the
+        // card's starting conditioning, and it must not add a round trip to the add.
+        // A missing or unreadable record is not a failure -- it just means the
+        // ordinary video default.
+        const [item, metadata] = await Promise.all([
+          galleryItems.resolve({ kind: 'video', name: videoName }),
+          galleryVideos.metadata(videoName).catch(() => null),
+        ]);
 
         if (item?.kind === 'video') {
-          addVideoItem(item);
+          addVideoItem(item, getDefaultReferenceConditioning(metadata));
         }
       } catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -585,15 +631,49 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({
     }),
     [imageCount, maxImages, maxVideos, references, videoCount]
   );
+  const addPickedVideo = useCallback(
+    (item: GalleryVideoItem) => {
+      // The card goes in SYNCHRONOUSLY and is corrected afterwards, rather than waiting on
+      // the metadata the way the drop and upload paths do. The picker stays open and judges
+      // each click against the reference list as it stands -- an add that had not landed yet
+      // would leave the tile pickable (a second click would duplicate it), leave the
+      // remaining count stale, and let two picks land in whichever order their fetches
+      // finished, which for references is a different generation.
+      const entry = addVideoItem(item);
+
+      if (!entry) {
+        return;
+      }
+
+      // The metadata is the only thing that tells a wrapped audio upload from footage. An
+      // unreadable record is not a failure -- the ordinary video default just stands.
+      void galleryVideos
+        .metadata(item.name)
+        .then((metadata) => {
+          const conditioning = getDefaultReferenceConditioning(metadata);
+
+          if (conditioning === entry.conditioning) {
+            return;
+          }
+          // Matched by identity, not index: a card the user has since edited is a different
+          // object and keeps their choice, and a removed one is simply no longer there.
+          onChange((current) =>
+            current.map((existing) => (existing === entry ? { ...entry, conditioning } : existing))
+          );
+        })
+        .catch(() => undefined);
+    },
+    [addVideoItem, onChange]
+  );
   const handlePick = useCallback(
     (item: GalleryItem) => {
       if (item.kind === 'video') {
-        addVideoItem(item);
+        addPickedVideo(item);
       } else {
         addImageReference(item);
       }
     },
-    [addImageReference, addVideoItem]
+    [addImageReference, addPickedVideo]
   );
 
   const updateReference = useCallback(
@@ -693,7 +773,7 @@ export const VideoReferenceListField = memo(function VideoReferenceListField({
       <Input accept={IMAGE_UPLOAD_ACCEPT} hidden ref={imageInputRef} type="file" onChange={handleImageFileChange} />
       {/* Audio files upload too: the server wraps them into waveform videos, which is
           how audio-only reference clips enter the pipeline. */}
-      <Input accept="video/*,audio/*" hidden ref={videoInputRef} type="file" onChange={handleVideoFileChange} />
+      <Input accept={MEDIA_UPLOAD_ACCEPT} hidden ref={videoInputRef} type="file" onChange={handleVideoFileChange} />
     </Stack>
   );
 });