Hold the current background until the next one is ready

.changeset/hold-background-until-ready.md

@@ -0,0 +1,5 @@
+---
+'@livekit/track-processors': patch
+---
+
+Keep the current blur or background image on screen until a new background image is ready, and show the camera instead of a black background when the blur radius is set to 0

src/transformers/BackgroundTransformer.ts

@@ -116,7 +116,11 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
       const imageData = await createImageBitmap(img);
       this.backgroundImageAndPath = { imageData, path };
     }
-    this.gl?.setBackgroundImage(this.backgroundImageAndPath.imageData);
+    // A later update() switched to another background while the image was loading
+    if (this.options.imagePath !== path) {
+      return;
+    }
+    await this.gl?.setBackgroundImage(this.backgroundImageAndPath.imageData);
   }
 
   async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {
@@ -217,12 +221,16 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
   async update(opts: BackgroundOptions) {
     this.options = { ...this.options, ...opts };
 
-    this.gl?.setBlurRadius(opts.blurRadius ?? null);
+    // The blur or the previous image stays on screen until the new image is ready
     if (opts.imagePath) {
       await this.loadAndSetBackground(opts.imagePath);
+      if (this.options.imagePath !== opts.imagePath) {
+        return;
+      }
     } else {
       this.gl?.setBackgroundImage(null);
     }
+    this.gl?.setBlurRadius(opts.blurRadius ?? null);
     this.gl?.setBackgroundDisabled(opts.backgroundDisabled ?? false);
   }
 

src/webgl/index.ts

@@ -154,7 +154,7 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
       resizeTexture(gl, texture, maskBufferWidth, maskBufferHeight);
     }
     if (backgroundSourceImage) {
-      // Not awaited: the placeholder background shows until the re-crop resolves, as on the initial set.
+      // Not awaited: the image cropped for the previous size shows until the re-crop resolves.
       setBackgroundImage(backgroundSourceImage);
     }
   }
@@ -183,8 +183,8 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
     gl.bindTexture(gl.TEXTURE_2D, frameTexture);
     gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
 
-    // Apply blur if enabled (and no custom background is set)
-    let backgroundTexture = bgTexture;
+    // The blurred frame, else the custom background, else the frame itself, which the composite passes through
+    let backgroundTexture = frameTexture;
 
     if (blurRadius) {
       const downSampledFrameTexture = applyDownsampling(
@@ -248,29 +248,28 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
    * @param image The background image to use, or null to clear
    */
   async function setBackgroundImage(image: ImageBitmap | null) {
-    // Clear existing background
     backgroundSourceImage = image;
-    customBackgroundImage = null;
 
-    if (image) {
-      customBackgroundImage = getEmptyImageData();
-      // Show the placeholder until the cropped image is ready: renderFrame no longer uploads it.
-      gl.activeTexture(gl.TEXTURE0);
-      gl.bindTexture(gl.TEXTURE_2D, bgTexture);
-      gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
+    if (!image) {
+      customBackgroundImage = null;
+    } else {
+      // bgTexture keeps showing the current image until the new one is cropped
+      let croppedImage: ImageBitmap | ImageData = getEmptyImageData();
       try {
         // Resize and crop the image to cover the canvas
-        const croppedImage = await resizeImageToCover(image, canvas.width, canvas.height);
-
-        // Store the cropped and resized image
-        customBackgroundImage = croppedImage;
+        croppedImage = await resizeImageToCover(image, canvas.width, canvas.height);
       } catch (error) {
         log.error(
           'Error processing background image, falling back to black background:',
           error,
         );
       }
+      // A later call or cleanup() replaced or cleared the image while it was cropping
+      if (image !== backgroundSourceImage) {
+        return;
+      }
 
+      customBackgroundImage = croppedImage;
       gl.activeTexture(gl.TEXTURE0);
       gl.bindTexture(gl.TEXTURE_2D, bgTexture);
       gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
@@ -279,7 +278,6 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
 
   function setBlurRadius(radius: number | null) {
     blurRadius = radius ? Math.max(1, Math.floor(radius / downsampleFactor)) : null; // we are downsampling the blur texture, so decrease the radius here for better performance with a similar visual result
-    setBackgroundImage(null);
   }
 
   function setBackgroundDisabled(disabled: boolean) {

test/webgl-pipeline.test.ts

@@ -137,6 +137,42 @@ describe('WebGL pipeline', () => {
     expect(samePixels(pixels(renderer.canvas), first), 'after blur').toBe(true);
   });
 
+  it('passes the frame through when the blur is cleared and no image is set', () => {
+    using renderer = createRenderer(1280, 720);
+    renderer.pipeline.setBlurRadius(10);
+    renderer.process(frame, mask);
+    renderer.pipeline.setBlurRadius(null);
+    renderer.process(frame, mask);
+
+    expect(samePixels(pixels(renderer.canvas), pixels(frame))).toBe(true);
+  });
+
+  it('passes the frame through once the background image is cleared', async () => {
+    using renderer = createRenderer(1280, 720);
+    await renderer.pipeline.setBackgroundImage(background);
+    renderer.process(frame, mask);
+    await renderer.pipeline.setBackgroundImage(null);
+    renderer.process(frame, mask);
+
+    expect(samePixels(pixels(renderer.canvas), pixels(frame))).toBe(true);
+  });
+
+  it('keeps showing the previous image until the next one is ready', async () => {
+    using renderer = createRenderer(1280, 720);
+    await renderer.pipeline.setBackgroundImage(background);
+    renderer.process(frame, mask);
+    const previous = pixels(renderer.canvas);
+
+    // The robot frame stands in for a second background image.
+    const next = renderer.pipeline.setBackgroundImage(frame);
+    renderer.process(frame, mask);
+    expect(samePixels(pixels(renderer.canvas), previous), 'while loading').toBe(true);
+
+    await next;
+    renderer.process(frame, mask);
+    expect(samePixels(pixels(renderer.canvas), previous), 'once loaded').toBe(false);
+  });
+
   it('passes the frame through untouched when the background is disabled', () => {
     using renderer = createRenderer(1280, 720);
     renderer.pipeline.setBlurRadius(10);