From 8983e1ea82e8303e61296ff5fe1bdc1f967fef56 Mon Sep 17 00:00:00 2001 From: Sen Chang Date: Tue, 20 Oct 2026 09:26:33 -0700 Subject: [PATCH] Hold the current background until the next one is ready Fixes #140. setBackgroundImage() writes bgTexture and customBackgroundImage only once the crop resolves, so a swap keeps showing the current image, and drops a crop whose image was replaced or cleared meanwhile. The transformer loads a new image before clearing the blur. With neither a blur nor an image, renderFrame composites against the frame itself, so a blur radius of 0 shows the camera instead of a black background. --- .changeset/hold-background-until-ready.md | 5 ++++ src/transformers/BackgroundTransformer.ts | 12 ++++++-- src/webgl/index.ts | 30 +++++++++---------- test/webgl-pipeline.test.ts | 36 +++++++++++++++++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) create mode 100644 .changeset/hold-background-until-ready.md diff --git a/.changeset/hold-background-until-ready.md b/.changeset/hold-background-until-ready.md new file mode 100644 index 0000000..8ac85fd --- /dev/null +++ b/.changeset/hold-background-until-ready.md @@ -0,0 +1,5 @@ +--- +'@livekit/track-processors': patch +--- + +Keep the current blur or background image on screen until a new background image is ready, and show the camera instead of a black background when the blur radius is set to 0 diff --git a/src/transformers/BackgroundTransformer.ts b/src/transformers/BackgroundTransformer.ts index 3a86816..adb1ea3 100644 --- a/src/transformers/BackgroundTransformer.ts +++ b/src/transformers/BackgroundTransformer.ts @@ -116,7 +116,11 @@ export default class BackgroundProcessor extends VideoTransformer) { @@ -217,12 +221,16 @@ export default class BackgroundProcessor extends VideoTransformer { resizeTexture(gl, texture, maskBufferWidth, maskBufferHeight); } if (backgroundSourceImage) { - // Not awaited: the placeholder background shows until the re-crop resolves, as on the initial set. + // Not awaited: the image cropped for the previous size shows until the re-crop resolves. setBackgroundImage(backgroundSourceImage); } } @@ -183,8 +183,8 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => { gl.bindTexture(gl.TEXTURE_2D, frameTexture); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame); - // Apply blur if enabled (and no custom background is set) - let backgroundTexture = bgTexture; + // The blurred frame, else the custom background, else the frame itself, which the composite passes through + let backgroundTexture = frameTexture; if (blurRadius) { const downSampledFrameTexture = applyDownsampling( @@ -248,29 +248,28 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => { * @param image The background image to use, or null to clear */ async function setBackgroundImage(image: ImageBitmap | null) { - // Clear existing background backgroundSourceImage = image; - customBackgroundImage = null; - if (image) { - customBackgroundImage = getEmptyImageData(); - // Show the placeholder until the cropped image is ready: renderFrame no longer uploads it. - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, bgTexture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage); + if (!image) { + customBackgroundImage = null; + } else { + // bgTexture keeps showing the current image until the new one is cropped + let croppedImage: ImageBitmap | ImageData = getEmptyImageData(); try { // Resize and crop the image to cover the canvas - const croppedImage = await resizeImageToCover(image, canvas.width, canvas.height); - - // Store the cropped and resized image - customBackgroundImage = croppedImage; + croppedImage = await resizeImageToCover(image, canvas.width, canvas.height); } catch (error) { log.error( 'Error processing background image, falling back to black background:', error, ); } + // A later call or cleanup() replaced or cleared the image while it was cropping + if (image !== backgroundSourceImage) { + return; + } + customBackgroundImage = croppedImage; gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, bgTexture); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage); @@ -279,7 +278,6 @@ export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => { function setBlurRadius(radius: number | null) { blurRadius = radius ? Math.max(1, Math.floor(radius / downsampleFactor)) : null; // we are downsampling the blur texture, so decrease the radius here for better performance with a similar visual result - setBackgroundImage(null); } function setBackgroundDisabled(disabled: boolean) { diff --git a/test/webgl-pipeline.test.ts b/test/webgl-pipeline.test.ts index a3eaa5e..351daf1 100644 --- a/test/webgl-pipeline.test.ts +++ b/test/webgl-pipeline.test.ts @@ -137,6 +137,42 @@ describe('WebGL pipeline', () => { expect(samePixels(pixels(renderer.canvas), first), 'after blur').toBe(true); }); + it('passes the frame through when the blur is cleared and no image is set', () => { + using renderer = createRenderer(1280, 720); + renderer.pipeline.setBlurRadius(10); + renderer.process(frame, mask); + renderer.pipeline.setBlurRadius(null); + renderer.process(frame, mask); + + expect(samePixels(pixels(renderer.canvas), pixels(frame))).toBe(true); + }); + + it('passes the frame through once the background image is cleared', async () => { + using renderer = createRenderer(1280, 720); + await renderer.pipeline.setBackgroundImage(background); + renderer.process(frame, mask); + await renderer.pipeline.setBackgroundImage(null); + renderer.process(frame, mask); + + expect(samePixels(pixels(renderer.canvas), pixels(frame))).toBe(true); + }); + + it('keeps showing the previous image until the next one is ready', async () => { + using renderer = createRenderer(1280, 720); + await renderer.pipeline.setBackgroundImage(background); + renderer.process(frame, mask); + const previous = pixels(renderer.canvas); + + // The robot frame stands in for a second background image. + const next = renderer.pipeline.setBackgroundImage(frame); + renderer.process(frame, mask); + expect(samePixels(pixels(renderer.canvas), previous), 'while loading').toBe(true); + + await next; + renderer.process(frame, mask); + expect(samePixels(pixels(renderer.canvas), previous), 'once loaded').toBe(false); + }); + it('passes the frame through untouched when the background is disabled', () => { using renderer = createRenderer(1280, 720); renderer.pipeline.setBlurRadius(10);