Video frames as standalone GPU textures

vendor/zed/PATCHES.md

@@ -5,3 +5,4 @@ Each local patch to this tree is a commit on the integrating branch, never on `z
 | patch | commit | files |
 |---|---|---|
 | `assets/fonts` | this commit | `vendor/zed/assets/fonts/` |
+| `video-frames` | this commit | `vendor/zed/crates/gpui/src/platform.rs`, `vendor/zed/crates/gpui/src/scene.rs`, `vendor/zed/crates/gpui/src/window.rs`, `vendor/zed/crates/gpui_wgpu/src/wgpu_renderer.rs`, `vendor/zed/crates/gpui_wgpu/src/shaders.wgsl`, `vendor/zed/crates/gpui_wgpu/src/shaders_storage.wgsl`, `vendor/zed/crates/gpui_wgpu/src/shaders_webgl.wgsl` |

vendor/zed/crates/gpui/src/platform.rs

@@ -1739,6 +1739,28 @@ impl From<RenderImageParams> for AtlasKey {
     }
 }
 
+/// A handle to a standalone GPU texture holding one video frame.
+///
+/// Video frames are full-frame images that arrive every frame, so they are
+/// not packed into the sprite atlas. The id is renderer-local and must be
+/// freed with [`Window::free_video_texture`](crate::Window::free_video_texture)
+/// once the frame is no longer painted.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[repr(transparent)]
+pub struct VideoTextureId(pub(crate) u64);
+
+impl VideoTextureId {
+    /// Build an id from the raw value stored on a painted frame.
+    pub fn new(id: u64) -> Self {
+        Self(id)
+    }
+
+    /// The raw value, for matching a painted frame back to its texture.
+    pub fn as_u64(self) -> u64 {
+        self.0
+    }
+}
+
 #[expect(missing_docs)]
 pub trait PlatformAtlas {
     /// The builder runs with the atlas locked and must not re-enter the same atlas.

vendor/zed/crates/gpui/src/scene.rs

@@ -49,6 +49,13 @@ pub struct Scene {
     pub monochrome_sprites: Vec<MonochromeSprite>,
     pub subpixel_sprites: Vec<SubpixelSprite>,
     pub polychrome_sprites: Vec<PolychromeSprite>,
+    pub video_frames: Vec<VideoFrame>,
+    /// The pixels behind [`Self::video_frames`], keyed by texture id.
+    ///
+    /// A frame is painted from a standalone texture the renderer owns. This map
+    /// carries the pixels the renderer has not uploaded yet, so a texture that
+    /// is not painted this frame is simply absent and keeps its GPU copy.
+    pub video_frame_pixels: std::collections::HashMap<u64, VideoFramePixels>,
     pub surfaces: Vec<PaintSurface>,
 }
 
@@ -65,6 +72,8 @@ impl Scene {
         self.monochrome_sprites.clear();
         self.subpixel_sprites.clear();
         self.polychrome_sprites.clear();
+        self.video_frames.clear();
+        self.video_frame_pixels.clear();
         self.surfaces.clear();
     }
 
@@ -129,6 +138,10 @@ impl Scene {
                 sprite.order = order;
                 self.polychrome_sprites.push(*sprite);
             }
+            Primitive::VideoFrame(video_frame) => {
+                video_frame.order = order;
+                self.video_frames.push(*video_frame);
+            }
             Primitive::Surface(surface) => {
                 surface.order = order;
                 self.surfaces.push(surface.clone());
@@ -159,6 +172,7 @@ impl Scene {
             .sort_by_key(|sprite| (sprite.order, sprite.tile.tile_id));
         self.polychrome_sprites
             .sort_by_key(|sprite| (sprite.order, sprite.tile.tile_id));
+        self.video_frames.sort_by_key(|frame| frame.order);
         self.surfaces.sort_by_key(|surface| surface.order);
     }
 
@@ -185,6 +199,8 @@ impl Scene {
             subpixel_sprites_iter: self.subpixel_sprites.iter().peekable(),
             polychrome_sprites_start: 0,
             polychrome_sprites_iter: self.polychrome_sprites.iter().peekable(),
+            video_frames_start: 0,
+            video_frames_iter: self.video_frames.iter().peekable(),
             surfaces_start: 0,
             surfaces_iter: self.surfaces.iter().peekable(),
         }
@@ -208,6 +224,7 @@ pub(crate) enum PrimitiveKind {
     MonochromeSprite,
     SubpixelSprite,
     PolychromeSprite,
+    VideoFrame,
     Surface,
 }
 
@@ -227,6 +244,7 @@ pub enum Primitive {
     MonochromeSprite(MonochromeSprite),
     SubpixelSprite(SubpixelSprite),
     PolychromeSprite(PolychromeSprite),
+    VideoFrame(VideoFrame),
     Surface(PaintSurface),
 }
 
@@ -241,6 +259,7 @@ impl Primitive {
             Primitive::MonochromeSprite(sprite) => &sprite.bounds,
             Primitive::SubpixelSprite(sprite) => &sprite.bounds,
             Primitive::PolychromeSprite(sprite) => &sprite.bounds,
+            Primitive::VideoFrame(video_frame) => &video_frame.bounds,
             Primitive::Surface(surface) => &surface.bounds,
         }
     }
@@ -254,6 +273,7 @@ impl Primitive {
             Primitive::MonochromeSprite(sprite) => &sprite.content_mask,
             Primitive::SubpixelSprite(sprite) => &sprite.content_mask,
             Primitive::PolychromeSprite(sprite) => &sprite.content_mask,
+            Primitive::VideoFrame(video_frame) => &video_frame.content_mask,
             Primitive::Surface(surface) => &surface.content_mask,
         }
     }
@@ -281,6 +301,8 @@ struct BatchIterator<'a> {
     subpixel_sprites_iter: Peekable<slice::Iter<'a, SubpixelSprite>>,
     polychrome_sprites_start: usize,
     polychrome_sprites_iter: Peekable<slice::Iter<'a, PolychromeSprite>>,
+    video_frames_start: usize,
+    video_frames_iter: Peekable<slice::Iter<'a, VideoFrame>>,
     surfaces_start: usize,
     surfaces_iter: Peekable<slice::Iter<'a, PaintSurface>>,
 }
@@ -312,6 +334,10 @@ impl<'a> Iterator for BatchIterator<'a> {
                 self.polychrome_sprites_iter.peek().map(|s| s.order),
                 PrimitiveKind::PolychromeSprite,
             ),
+            (
+                self.video_frames_iter.peek().map(|s| s.order),
+                PrimitiveKind::VideoFrame,
+            ),
             (
                 self.surfaces_iter.peek().map(|s| s.order),
                 PrimitiveKind::Surface,
@@ -447,6 +473,20 @@ impl<'a> Iterator for BatchIterator<'a> {
                     range: sprites_start..sprites_end,
                 })
             }
+            PrimitiveKind::VideoFrame => {
+                let frames_start = self.video_frames_start;
+                let mut frames_end = frames_start + 1;
+                self.video_frames_iter.next();
+                while self
+                    .video_frames_iter
+                    .next_if(|frame| (frame.order, batch_kind) < max_order_and_kind)
+                    .is_some()
+                {
+                    frames_end += 1;
+                }
+                self.video_frames_start = frames_end;
+                Some(PrimitiveBatch::VideoFrames(frames_start..frames_end))
+            }
             PrimitiveKind::Surface => {
                 let surfaces_start = self.surfaces_start;
                 let mut surfaces_end = surfaces_start + 1;
@@ -492,6 +532,7 @@ pub enum PrimitiveBatch {
         texture_id: AtlasTextureId,
         range: Range<usize>,
     },
+    VideoFrames(Range<usize>),
     Surfaces(Range<usize>),
 }
 
@@ -524,6 +565,7 @@ impl PrimitiveBatch {
                     texture_id.index
                 )
             }
+            Self::VideoFrames(range) => format!("video frames ({})", range.len()),
             Self::Surfaces(range) => format!("surfaces ({})", range.len()),
         }
     }
@@ -763,6 +805,42 @@ impl From<PolychromeSprite> for Primitive {
     }
 }
 
+/// The pixels of one video texture, in tightly packed BGRA8.
+#[derive(Clone, Debug)]
+#[expect(missing_docs)]
+pub struct VideoFramePixels {
+    pub width: u32,
+    pub height: u32,
+    pub pixels: std::sync::Arc<[u8]>,
+    /// Bumped on every upload, so the renderer re-uploads only when it changes.
+    pub generation: u64,
+}
+
+/// A video frame painted from a standalone GPU texture.
+///
+/// Unlike [`PolychromeSprite`], the texture is not an atlas tile: the whole
+/// texture is sampled, clipped by [`Self::content_mask`] and rounded by
+/// [`Self::corner_radii`]. The texture itself lives in the renderer, keyed by
+/// [`Self::texture_id`].
+#[derive(Copy, Clone, Debug)]
+#[repr(C)]
+#[expect(missing_docs)]
+pub struct VideoFrame {
+    pub order: DrawOrder,
+    pub bounds: Bounds<ScaledPixels>,
+    pub content_mask: ContentMask<ScaledPixels>,
+    pub corner_radii: Corners<ScaledPixels>,
+    pub texture_id: u64,
+    pub opacity: f32,
+    pub pad: u32,
+}
+
+impl From<VideoFrame> for Primitive {
+    fn from(video_frame: VideoFrame) -> Self {
+        Primitive::VideoFrame(video_frame)
+    }
+}
+
 #[derive(Clone, Debug)]
 #[allow(missing_docs)]
 pub struct PaintSurface {

vendor/zed/crates/gpui/src/window.rs

@@ -20,9 +20,10 @@ use crate::{
     StrikethroughStyle, Style, SubpixelSprite, SubscriberSet, Subscription, SystemWindowTab,
     SystemWindowTabController, TabStopMap, TaffyLayoutEngine, Task, TextInputConfiguration,
     TextInputStateChange, TextRenderingMode, TextStyle, TextStyleRefinement, ThermalState,
-    TransformationMatrix, Underline, UnderlineStyle, WindowAppearance, WindowBackgroundAppearance,
-    WindowBounds, WindowControls, WindowDecorations, WindowOptions, WindowParams, WindowTextSystem,
-    WindowVisibility, point, prelude::*, px, rems, size, transparent_black,
+    TransformationMatrix, Underline, UnderlineStyle, VideoTextureId, WindowAppearance,
+    WindowBackgroundAppearance, WindowBounds, WindowControls, WindowDecorations, WindowOptions,
+    WindowParams, WindowTextSystem, WindowVisibility, point, prelude::*, px, rems, size,
+    transparent_black,
 };
 
 use crate::gestures::{GestureTuning, RecognizedTouchGesture, TouchGestureRecognizer};
@@ -1223,6 +1224,13 @@ pub struct Window {
     is_resizable: bool,
     is_minimizable: bool,
     sprite_atlas: Arc<dyn PlatformAtlas>,
+    /// Standalone GPU textures for video frames, keyed by [`VideoTextureId`].
+    ///
+    /// Each entry is the latest BGRA8 frame the renderer should upload. Entries
+    /// are removed only by [`Self::free_video_texture`], so a texture that is
+    /// not painted this frame keeps its pixels for the next one.
+    video_textures: Rc<RefCell<FxHashMap<VideoTextureId, VideoTexture>>>,
+    next_video_texture_id: Rc<Cell<u64>>,
     text_system: Arc<WindowTextSystem>,
     text_rendering_mode: Rc<Cell<TextRenderingMode>>,
     rem_size: Pixels,
@@ -1384,6 +1392,14 @@ impl<'a> PendingInputStatus<'a> {
     }
 }
 
+/// One video frame held until the renderer uploads it.
+struct VideoTexture {
+    width: u32,
+    height: u32,
+    pixels: std::sync::Arc<[u8]>,
+    generation: u64,
+}
+
 /// The timeout state for pending input.
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub struct PendingInputTimeoutStatus {
@@ -2131,6 +2147,8 @@ impl Window {
             is_resizable,
             is_minimizable,
             sprite_atlas,
+            video_textures: Rc::new(RefCell::new(FxHashMap::default())),
+            next_video_texture_id: Rc::new(Cell::new(1)),
             text_system,
             text_rendering_mode: cx.text_rendering_mode.clone(),
             rem_size: px(16.),
@@ -5171,6 +5189,109 @@ impl Window {
         });
     }
 
+    /// Allocate a standalone GPU texture for a video frame.
+    ///
+    /// The texture is not part of the sprite atlas: video frames are full-frame
+    /// images replaced every frame, and packing them into the atlas would evict
+    /// glyphs and images on every frame. Call [`Self::upload_video_frame`] to
+    /// replace its pixels and [`Self::paint_video_frame`] to draw it. Free it
+    /// with [`Self::free_video_texture`] when the frame is no longer shown.
+    pub fn alloc_video_texture(&mut self, width: u32, height: u32) -> VideoTextureId {
+        let id = VideoTextureId(post_inc(&mut self.next_video_texture_id.get()));
+        self.video_textures.borrow_mut().insert(
+            id,
+            VideoTexture {
+                width,
+                height,
+                pixels: std::sync::Arc::from([]),
+                generation: 0,
+            },
+        );
+        id
+    }
+
+    /// Replace the pixels of a video texture.
+    ///
+    /// `pixels` is tightly packed BGRA8, so its length must be `width * height *
+    /// 4` of the size passed to [`Self::alloc_video_texture`]. The upload is
+    /// applied on the next frame; this does not paint anything by itself.
+    pub fn upload_video_frame(&mut self, id: VideoTextureId, pixels: Vec<u8>) {
+        let mut textures = self.video_textures.borrow_mut();
+        let Some(texture) = textures.get_mut(&id) else {
+            return;
+        };
+        let expected = texture.width as usize * texture.height as usize * 4;
+        if pixels.len() != expected {
+            log::warn!(
+                "video frame upload of {} bytes does not match texture {}x{} ({} bytes); dropped",
+                pixels.len(),
+                texture.width,
+                texture.height,
+                expected,
+            );
+            return;
+        }
+        texture.pixels = pixels.into();
+        texture.generation += 1;
+    }
+
+    /// Paint a video texture into the current frame.
+    ///
+    /// The whole texture is sampled into `bounds`, clipped by the current
+    /// content mask and rounded by `corner_radii`. A texture that was freed,
+    /// or that has not had a frame uploaded yet, paints nothing.
+    pub fn paint_video_frame(
+        &mut self,
+        bounds: Bounds<Pixels>,
+        corner_radii: Corners<Pixels>,
+        id: VideoTextureId,
+    ) {
+        use crate::VideoFrame;
+
+        self.invalidator.debug_assert_paint();
+
+        if !self
+            .video_textures
+            .borrow()
+            .get(&id)
+            .is_some_and(|texture| !texture.pixels.is_empty())
+        {
+            return;
+        }
+
+        self.next_frame.scene.insert_primitive(VideoFrame {
+            order: 0,
+            bounds: self.snap_bounds(bounds),
+            content_mask: self.snapped_content_mask(),
+            corner_radii: corner_radii.scale(self.scale_factor()),
+            texture_id: id.0,
+            opacity: self.element_opacity(),
+            pad: 0,
+        });
+        if let Some(texture) = self.video_textures.borrow().get(&id) {
+            use crate::VideoFramePixels;
+
+            self.next_frame.scene.video_frame_pixels.insert(
+                id.0,
+                VideoFramePixels {
+                    width: texture.width,
+                    height: texture.height,
+                    pixels: texture.pixels.clone(),
+                    generation: texture.generation,
+                },
+            );
+        }
+        self.request_animation_frame();
+    }
+
+    /// Drop a video texture allocated with [`Self::alloc_video_texture`].
+    ///
+    /// The renderer releases the GPU texture on its next frame. Painting the id
+    /// afterwards does nothing.
+    pub fn free_video_texture(&mut self, id: VideoTextureId) {
+        self.video_textures.borrow_mut().remove(&id);
+    }
+
     /// Removes an image from the sprite atlas.
     pub fn drop_image(&mut self, data: Arc<RenderImage>) -> Result<()> {
         for frame_index in 0..data.frame_count() {