From 5b1a0307535886540b92f4588bd91b1a80c066d3 Mon Sep 17 00:00:00 2001 From: Vinicius Deolindo Date: Tue, 13 Oct 2026 00:16:44 -0300 Subject: [PATCH 1/7] luma: don't convert opaque frames to rgba for fix_background_alpha Previously, a dissolve between two opaque 4K60 H.264 clips at 100 Mbit/s ran at 8.3 fps on a 16-core desktop, when played on one thread as the Kdenlive monitor does. Most of this time went to color conversions that changed nothing. Kdenlive sets fix_background_alpha on every dissolve, and with this property, the luma transition requested both frames as rgba and converted the result back to yuv422. It did this only to make the padding that filter_resize adds transparent, so for opaque video, the four conversions had no effect. The round trip also slowed down the work after the transition, because it left the result with an alpha mask that was fully opaque. Kdenlive composites its tracks with a qtblend transition, which saw this mask and so composited the frame in rgba too. The transition now requests yuv422 and marks both frames with fix_background_alpha. The padding stays transparent, because when filter_resize pads a marked frame that has no alpha mask, it first adds an opaque one. The alpha of the result also stays correct: when one frame is translucent, the transition gives the other frame an opaque mask if it has none, as the round trip did, so that alpha_over can still compute it. With this change, the same dissolve runs at 73 fps. Its luma is also closer to the source, because the rgba round trip clipped colors that rgb cannot represent. In the first frame of the dissolve, luma is now identical to the decoded source, and before, the two differed by a PSNR of 38 dB. Apart from this, the output does not change. These cases give the same result as before: - A 16:9 clip in a 4:3 profile gets transparent padding. - With alpha_over, a dissolve into a clip with transparency shows the track below through it, with the same colors in every frame, also with a luma map and with invert. - With fix_background_alpha off, the output is bit-identical. --- src/modules/core/filter_resize.c | 11 ++++++ src/modules/core/transition_luma.c | 63 +++++++++++++++++++++--------- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/modules/core/filter_resize.c b/src/modules/core/filter_resize.c index 63fca69ff7..fc61bdd6ca 100644 --- a/src/modules/core/filter_resize.c +++ b/src/modules/core/filter_resize.c @@ -176,6 +176,17 @@ static uint8_t *frame_resize_image(mlt_frame frame, int owidth, int oheight, mlt // Now update the frame mlt_frame_set_image(frame, output, owidth * (oheight + 1) * bpp, mlt_pool_release); + // An opaque image without an alpha mask gets one, so that the padding can be + // transparent. This gives the same result as padding an rgba image, without + // converting the image to rgba and back. + if (format == mlt_image_yuv422 && !alpha + && mlt_properties_get_int(properties, "fix_background_alpha")) { + alpha_size = iwidth * iheight; + alpha = mlt_pool_alloc(alpha_size); + memset(alpha, 255, alpha_size); + mlt_frame_set_alpha(frame, alpha, alpha_size, mlt_pool_release); + } + // We should resize the alpha too if (format != mlt_image_rgba && alpha && alpha_size >= iwidth * iheight) { alpha = resize_alpha(alpha, owidth, oheight, iwidth, iheight, alpha_value); diff --git a/src/modules/core/transition_luma.c b/src/modules/core/transition_luma.c index c8eb438be8..4bd9d4ae85 100644 --- a/src/modules/core/transition_luma.c +++ b/src/modules/core/transition_luma.c @@ -128,6 +128,16 @@ static int dissolve_slice_yuv422(int id, int index, int count, void *context) return 0; } +// Gives a frame an opaque alpha mask, as converting it to rgba and back does. +static uint8_t *add_opaque_alpha(mlt_frame frame, int width, int height) +{ + int size = width * height; + uint8_t *alpha = mlt_pool_alloc(size); + memset(alpha, 255, size); + mlt_frame_set_alpha(frame, alpha, size, mlt_pool_release); + return alpha; +} + static inline int dissolve_yuv422(mlt_frame frame, mlt_frame that, float weight, @@ -140,27 +150,35 @@ static inline int dissolve_yuv422(mlt_frame frame, int ret = 0; int i = height + 1; int width_src = width, height_src = height; - mlt_image_format format = (fix_background_alpha && mlt_frame_has_convert_image(frame)) - ? mlt_image_rgba - : mlt_image_yuv422; + mlt_image_format format = mlt_image_yuv422; uint8_t *p_src, *p_dest; uint8_t *alpha_src; uint8_t *alpha_dst; int mix = weight * (1 << 16); + if (fix_background_alpha) { + // Tell filter_resize to make the padding transparent (see filter_resize.c). + mlt_properties_set_int(MLT_FRAME_PROPERTIES(frame), "fix_background_alpha", 1); + mlt_properties_set_int(MLT_FRAME_PROPERTIES(that), "fix_background_alpha", 1); + } + mlt_frame_get_image(frame, &p_dest, &format, &width, &height, 1); - if (fix_background_alpha && mlt_frame_has_convert_image(frame)) - mlt_frame_convert_image(frame, &p_dest, &format, mlt_image_yuv422); alpha_dst = mlt_frame_get_alpha(frame); - if (fix_background_alpha && mlt_frame_has_convert_image(that)) - format = mlt_image_rgba; + format = mlt_image_yuv422; mlt_frame_get_image(that, &p_src, &format, &width_src, &height_src, 0); - if (mlt_frame_has_convert_image(that)) - mlt_frame_convert_image(that, &p_src, &format, mlt_image_yuv422); alpha_src = mlt_frame_get_alpha(that); int is_translucent = (alpha_dst && !is_opaque(alpha_dst, width, height)) || (alpha_src && !is_opaque(alpha_src, width_src, height_src)); + // An opaque frame without an alpha mask must get one when the other frame is + // translucent, so that the alpha of the result is computed (see alpha_over). + if (fix_background_alpha && is_translucent) { + if (!alpha_dst) + alpha_dst = add_opaque_alpha(frame, width, height); + if (!alpha_src) + alpha_src = add_opaque_alpha(that, width_src, height_src); + } + // Pick the lesser of two evils ;-) width_src = width_src > width ? width : width_src; height_src = height_src > height ? height : height_src; @@ -341,29 +359,27 @@ static void luma_composite_yuv422(mlt_frame a_frame, { int width_src = *width, height_src = *height; int width_dest = *width, height_dest = *height; - mlt_image_format format_src = (fix_background_alpha && mlt_frame_has_convert_image(a_frame)) - ? mlt_image_rgba - : mlt_image_yuv422; - mlt_image_format format_dest = (fix_background_alpha && mlt_frame_has_convert_image(b_frame)) - ? mlt_image_rgba - : mlt_image_yuv422; + mlt_image_format format_src = mlt_image_yuv422; + mlt_image_format format_dest = mlt_image_yuv422; uint8_t *p_src, *p_dest; uint8_t *alpha_src, *alpha_dest; int i, j; int stride_src; int stride_dest; + if (fix_background_alpha) { + // Tell filter_resize to make the padding transparent (see filter_resize.c). + mlt_properties_set_int(MLT_FRAME_PROPERTIES(a_frame), "fix_background_alpha", 1); + mlt_properties_set_int(MLT_FRAME_PROPERTIES(b_frame), "fix_background_alpha", 1); + } + if (mlt_properties_get(&a_frame->parent, "distort")) mlt_properties_set(&b_frame->parent, "distort", mlt_properties_get(&a_frame->parent, "distort")); mlt_frame_get_image(a_frame, &p_dest, &format_dest, &width_dest, &height_dest, 1); - if (fix_background_alpha && mlt_frame_has_convert_image(a_frame)) - mlt_frame_convert_image(a_frame, &p_dest, &format_dest, mlt_image_yuv422); alpha_dest = mlt_frame_get_alpha(a_frame); mlt_frame_get_image(b_frame, &p_src, &format_src, &width_src, &height_src, 0); - if (fix_background_alpha && mlt_frame_has_convert_image(b_frame)) - mlt_frame_convert_image(b_frame, &p_src, &format_src, mlt_image_yuv422); alpha_src = mlt_frame_get_alpha(b_frame); if (*width == 0 || *height == 0) @@ -372,6 +388,15 @@ static void luma_composite_yuv422(mlt_frame a_frame, int is_translucent = (alpha_dest && !is_opaque(alpha_dest, width_dest, height_dest)) || (alpha_src && !is_opaque(alpha_src, width_src, height_src)); + // An opaque frame without an alpha mask must get one when the other frame is + // translucent, so that the alpha of the result is computed (see alpha_over). + if (fix_background_alpha && is_translucent) { + if (!alpha_dest) + alpha_dest = add_opaque_alpha(a_frame, width_dest, height_dest); + if (!alpha_src) + alpha_src = add_opaque_alpha(b_frame, width_src, height_src); + } + // Pick the lesser of two evils ;-) width_src = width_src > width_dest ? width_dest : width_src; height_src = height_src > height_dest ? height_dest : height_src; From 38299b2b71dfcc7dbc176e2ac051c52b6b508132 Mon Sep 17 00:00:00 2001 From: Vinicius Deolindo Date: Tue, 13 Oct 2026 00:16:56 -0300 Subject: [PATCH 2/7] luma: dissolve opaque yuv422 frames in slices Previously, the luma transition split the dissolve of translucent frames into slices, but it mixed opaque frames line by line on the calling thread. For 4K frames, this loop took a large part of the time of each frame. Opaque frames now use slices too, as translucent frames do. The output does not change, and on a 16-core desktop, a dissolve between two 4K60 H.264 clips at 100 Mbit/s goes from 73 to 77 fps. --- src/modules/core/transition_luma.c | 63 +++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/src/modules/core/transition_luma.c b/src/modules/core/transition_luma.c index 4bd9d4ae85..397eb2607f 100644 --- a/src/modules/core/transition_luma.c +++ b/src/modules/core/transition_luma.c @@ -138,6 +138,50 @@ static uint8_t *add_opaque_alpha(mlt_frame frame, int width, int height) return alpha; } +struct dissolve_opaque_context +{ + uint8_t *dst_image; + uint8_t *src_image; + uint8_t *dst_alpha; + uint8_t *src_alpha; + int dst_width; + int src_width; + int height; + int mix; +}; + +static int dissolve_opaque_slice_yuv422(int id, int index, int count, void *context) +{ + struct dissolve_opaque_context ctx = *((struct dissolve_opaque_context *) context); + int slice_start, slice_height = mlt_slices_size_slice(count, index, ctx.height, &slice_start); + + ctx.dst_image += slice_start * (ctx.dst_width << 1); + ctx.src_image += slice_start * (ctx.src_width << 1); + if (ctx.dst_alpha) + ctx.dst_alpha += slice_start * ctx.dst_width; + if (ctx.src_alpha) + ctx.src_alpha += slice_start * ctx.src_width; + + for (int i = 0; i < slice_height; i++) { + composite_line_yuv(ctx.dst_image, + ctx.src_image, + ctx.src_width, + ctx.src_alpha, + ctx.dst_alpha, + ctx.mix, + NULL, + 0, + 0); + ctx.dst_image += ctx.dst_width << 1; + ctx.src_image += ctx.src_width << 1; + if (ctx.dst_alpha) + ctx.dst_alpha += ctx.dst_width; + if (ctx.src_alpha) + ctx.src_alpha += ctx.src_width; + } + return 0; +} + static inline int dissolve_yuv422(mlt_frame frame, mlt_frame that, float weight, @@ -148,7 +192,6 @@ static inline int dissolve_yuv422(mlt_frame frame, int fix_background_alpha) { int ret = 0; - int i = height + 1; int width_src = width, height_src = height; mlt_image_format format = mlt_image_yuv422; uint8_t *p_src, *p_dest; @@ -193,15 +236,15 @@ static inline int dissolve_yuv422(mlt_frame frame, .weight = weight}; mlt_slices_run_normal(threads, dissolve_slice_yuv422, &context); } else { - while (--i) { - composite_line_yuv(p_dest, p_src, width_src, alpha_src, alpha_dst, mix, NULL, 0, 0); - p_src += width_src << 1; - p_dest += width << 1; - if (alpha_src) - alpha_src += width_src; - if (alpha_dst) - alpha_dst += width; - } + struct dissolve_opaque_context context = {.dst_image = p_dest, + .src_image = p_src, + .dst_alpha = alpha_dst, + .src_alpha = alpha_src, + .dst_width = width, + .src_width = width_src, + .height = height, + .mix = mix}; + mlt_slices_run_normal(threads, dissolve_opaque_slice_yuv422, &context); } return ret; From 951c58205e9f788f52f77fb2f22b69c95daf18ee Mon Sep 17 00:00:00 2001 From: Vinicius Deolindo Date: Tue, 13 Oct 2026 00:16:56 -0300 Subject: [PATCH 3/7] avformat: convert yuv420p to yuyv422 without libswscale Previously, the conversion of progressive 8-bit 4:2:0 video to yuyv422 was slow, and it put the chroma half a line too low. The producer does this conversion on every frame of most video, and libswscale caused both problems: - MLT interpolates chroma vertically (SWS_BILINEAR), so libswscale ran its general scaler for this conversion. The producer also created a new scaler for each column slice of each frame. - libswscale copied chroma line n to output line 2n+1, and averaged two chroma lines for the even lines, whatever src_v_chr_pos said. The producer now does this conversion itself. In 4:2:0 video such as H.264 and MPEG-2, each chroma line lies between two luma lines, so each output line mixes the nearest chroma line (3/4) with the next nearest one (1/4). The conversion runs in line slices and uses SSE2 where available. The new conversion does not convert between YUV matrices, but libswscale does, for example for SD video in an HD profile. So the producer uses the new conversion only when the source and the destination use the same matrix. For this check, the choice of the libswscale matrix moves into mlt_get_sws_coefficients(). Other source formats also still use libswscale. Because the chroma is now in the correct position, the output changes where the chroma changes from one line to the next. At the sharp color edges of a test pattern, it changes by up to 113 levels. Luma does not change, and SD clips (tagged BT.601 or untagged) in an HD profile still use libswscale, so their output is bit-identical to before. On a 16-core desktop, a dissolve between two 4K60 H.264 clips at 100 Mbit/s goes from 77 to 104 fps. --- src/modules/avformat/common.c | 61 ++++++----------- src/modules/avformat/common.h | 1 + src/modules/avformat/producer_avformat.c | 84 +++++++++++++++++++++++- 3 files changed, 104 insertions(+), 42 deletions(-) diff --git a/src/modules/avformat/common.c b/src/modules/avformat/common.c index 96b5988f5e..c9253ab6a2 100644 --- a/src/modules/avformat/common.c +++ b/src/modules/avformat/common.c @@ -239,60 +239,41 @@ mlt_channel_layout mlt_get_channel_layout_or_default(const char *name, int chann return layout; } +const int *mlt_get_sws_coefficients(mlt_colorspace colorspace) +{ + switch (colorspace) { + case mlt_colorspace_smpte170m: + case mlt_colorspace_bt470bg: + case mlt_colorspace_bt601: + return sws_getCoefficients(SWS_CS_ITU601); + case mlt_colorspace_smpte240m: + return sws_getCoefficients(SWS_CS_SMPTE240M); + case mlt_colorspace_bt709: + return sws_getCoefficients(SWS_CS_ITU709); + case mlt_colorspace_bt2020_ncl: + case mlt_colorspace_bt2020_cl: + return sws_getCoefficients(SWS_CS_BT2020); + default: + return sws_getCoefficients(SWS_CS_DEFAULT); + } +} + int mlt_set_luma_transfer(struct SwsContext *context, mlt_colorspace src_colorspace, mlt_colorspace dst_colorspace, int src_full_range, int dst_full_range) { - const int *src_coefficients = sws_getCoefficients(SWS_CS_DEFAULT); - const int *dst_coefficients = sws_getCoefficients(SWS_CS_DEFAULT); int brightness = 0; int contrast = 1 << 16; int saturation = 1 << 16; int src_range = src_full_range ? 1 : 0; int dst_range = dst_full_range ? 1 : 0; - switch (src_colorspace) { - case mlt_colorspace_smpte170m: - case mlt_colorspace_bt470bg: - case mlt_colorspace_bt601: - src_coefficients = sws_getCoefficients(SWS_CS_ITU601); - break; - case mlt_colorspace_smpte240m: - src_coefficients = sws_getCoefficients(SWS_CS_SMPTE240M); - break; - case mlt_colorspace_bt709: - src_coefficients = sws_getCoefficients(SWS_CS_ITU709); - break; - case mlt_colorspace_bt2020_ncl: - case mlt_colorspace_bt2020_cl: - src_coefficients = sws_getCoefficients(SWS_CS_BT2020); - default: - break; - } - switch (dst_colorspace) { - case mlt_colorspace_smpte170m: - case mlt_colorspace_bt470bg: - case mlt_colorspace_bt601: - dst_coefficients = sws_getCoefficients(SWS_CS_ITU601); - break; - case mlt_colorspace_smpte240m: - dst_coefficients = sws_getCoefficients(SWS_CS_SMPTE240M); - break; - case mlt_colorspace_bt709: - dst_coefficients = sws_getCoefficients(SWS_CS_ITU709); - break; - case mlt_colorspace_bt2020_ncl: - case mlt_colorspace_bt2020_cl: - dst_coefficients = sws_getCoefficients(SWS_CS_BT2020); - default: - break; - } return sws_setColorspaceDetails(context, - src_coefficients, + mlt_get_sws_coefficients(src_colorspace), src_range, - dst_coefficients, + mlt_get_sws_coefficients(dst_colorspace), dst_range, brightness, contrast, diff --git a/src/modules/avformat/common.h b/src/modules/avformat/common.h index 8f1cf75ff9..b79593444a 100644 --- a/src/modules/avformat/common.h +++ b/src/modules/avformat/common.h @@ -36,6 +36,7 @@ mlt_channel_layout av_channel_layout_to_mlt(AVChannelLayout *layout); mlt_channel_layout av_channel_layout_to_mlt(int64_t layout); #endif mlt_channel_layout mlt_get_channel_layout_or_default(const char *name, int channels); +const int *mlt_get_sws_coefficients(mlt_colorspace colorspace); int mlt_set_luma_transfer(struct SwsContext *context, mlt_colorspace src_colorspace, mlt_colorspace dst_colorspace, diff --git a/src/modules/avformat/producer_avformat.c b/src/modules/avformat/producer_avformat.c index 1790b8aff2..87552799f0 100644 --- a/src/modules/avformat/producer_avformat.c +++ b/src/modules/avformat/producer_avformat.c @@ -59,6 +59,9 @@ // System header files #include +#ifdef __SSE2__ +#include +#endif #include #include #include @@ -2241,6 +2244,72 @@ static int sliced_h_pix_fmt_conv_proc(int id, int idx, int jobs, void *cookie) return 0; } +/** Convert progressive 8-bit 4:2:0 to packed 4:2:2 without libswscale. + * + * In 4:2:0 video such as H.264 and MPEG-2, each chroma line lies between two luma + * lines, so each output line mixes the nearest chroma line (3/4) and the next nearest + * one (1/4). For this conversion, libswscale runs its general scaler, which is much + * slower than this loop, and it places the chroma half a line too low. + */ +static int sliced_yuv420p_to_yuyv422_proc(int id, int idx, int jobs, void *cookie) +{ + struct sliced_pix_fmt_conv_t *ctx = (struct sliced_pix_fmt_conv_t *) cookie; + AVFrame *frame = ctx->frame; + int half_width = ctx->width / 2; + int chroma_height = (ctx->height + 1) / 2; + int start; + int lines = mlt_slices_size_slice(jobs, idx, ctx->height, &start); + + for (int y = start; y < start + lines; y++) { + int near_line = y / 2; + int far_line = (y & 1) ? FFMIN(near_line + 1, chroma_height - 1) : FFMAX(near_line - 1, 0); + const uint8_t *restrict luma = frame->data[0] + y * frame->linesize[0]; + const uint8_t *restrict u_near = frame->data[1] + near_line * frame->linesize[1]; + const uint8_t *restrict u_far = frame->data[1] + far_line * frame->linesize[1]; + const uint8_t *restrict v_near = frame->data[2] + near_line * frame->linesize[2]; + const uint8_t *restrict v_far = frame->data[2] + far_line * frame->linesize[2]; + uint8_t *restrict out = ctx->out_data[0] + y * ctx->out_stride[0]; + + int x = 0; +#ifdef __SSE2__ + // 32 pixels at a time: mix 16 chroma samples of each plane in 16 bits, + // then interleave them with the luma. + const __m128i zero = _mm_setzero_si128(); + const __m128i two = _mm_set1_epi16(2); + for (; x + 16 <= half_width; x += 16) { + __m128i un = _mm_loadu_si128((const __m128i *) (u_near + x)); + __m128i uf = _mm_loadu_si128((const __m128i *) (u_far + x)); + __m128i vn = _mm_loadu_si128((const __m128i *) (v_near + x)); + __m128i vf = _mm_loadu_si128((const __m128i *) (v_far + x)); +#define MIX(n, f) \ + _mm_srli_epi16(_mm_add_epi16(_mm_add_epi16(_mm_add_epi16(n, n), n), _mm_add_epi16(f, two)), 2) + __m128i u + = _mm_packus_epi16(MIX(_mm_unpacklo_epi8(un, zero), _mm_unpacklo_epi8(uf, zero)), + MIX(_mm_unpackhi_epi8(un, zero), _mm_unpackhi_epi8(uf, zero))); + __m128i v + = _mm_packus_epi16(MIX(_mm_unpacklo_epi8(vn, zero), _mm_unpacklo_epi8(vf, zero)), + MIX(_mm_unpackhi_epi8(vn, zero), _mm_unpackhi_epi8(vf, zero))); +#undef MIX + __m128i uv_lo = _mm_unpacklo_epi8(u, v); + __m128i uv_hi = _mm_unpackhi_epi8(u, v); + __m128i y_lo = _mm_loadu_si128((const __m128i *) (luma + 2 * x)); + __m128i y_hi = _mm_loadu_si128((const __m128i *) (luma + 2 * x + 16)); + _mm_storeu_si128((__m128i *) (out + 4 * x), _mm_unpacklo_epi8(y_lo, uv_lo)); + _mm_storeu_si128((__m128i *) (out + 4 * x + 16), _mm_unpackhi_epi8(y_lo, uv_lo)); + _mm_storeu_si128((__m128i *) (out + 4 * x + 32), _mm_unpacklo_epi8(y_hi, uv_hi)); + _mm_storeu_si128((__m128i *) (out + 4 * x + 48), _mm_unpackhi_epi8(y_hi, uv_hi)); + } +#endif + for (; x < half_width; x++) { + out[4 * x] = luma[2 * x]; + out[4 * x + 1] = (3 * u_near[x] + u_far[x] + 2) >> 2; + out[4 * x + 2] = luma[2 * x + 1]; + out[4 * x + 3] = (3 * v_near[x] + v_far[x] + 2) >> 2; + } + } + return 0; +} + static mlt_colorspace convert_image_yuvp(producer_avformat self, mlt_profile profile, AVFrame *frame, @@ -2598,8 +2667,19 @@ static void convert_image(producer_avformat self, c = (width + ctx.slice_w - 1) / ctx.slice_w; int last_slice_w = width - ctx.slice_w * (c - 1); - if (sliced && (last_slice_w % 8) == 0 - && !(ctx.src_format == AV_PIX_FMT_YUV422P && last_slice_w % 16)) { +#if LIBAVUTIL_VERSION_INT >= ((58 << 16) + (7 << 8) + 100) + int interlaced = frame->flags & AV_FRAME_FLAG_INTERLACED; +#else + int interlaced = frame->interlaced_frame; +#endif + if (sliced && !interlaced && width % 2 == 0 + && (src_pix_fmt == AV_PIX_FMT_YUV420P || src_pix_fmt == AV_PIX_FMT_YUVJ420P) + && self->full_range == dst_full_range + && mlt_get_sws_coefficients(ctx.src_colorspace) + == mlt_get_sws_coefficients(ctx.dst_colorspace)) { + mlt_slices_run_normal(0, sliced_yuv420p_to_yuyv422_proc, &ctx); + } else if (sliced && (last_slice_w % 8) == 0 + && !(ctx.src_format == AV_PIX_FMT_YUV422P && last_slice_w % 16)) { #if LIBAVUTIL_VERSION_INT >= ((58 << 16) + (7 << 8) + 100) c *= (frame->flags & AV_FRAME_FLAG_INTERLACED) ? 2 : 1; #else From 3be22b70621c7dad4a3e33804be5140168fa7402 Mon Sep 17 00:00:00 2001 From: Vinicius Deolindo Date: Tue, 13 Oct 2026 00:16:56 -0300 Subject: [PATCH 4/7] avformat: copy the probed stream information to the audio context Previously, when playback reached a clip whose file was not open yet, it stopped while MLT probed the file three times. Kdenlive opens files only at this point, so this happened at the start of clips. For a seekable file with audio and video, producer_open() called avformat_find_stream_info() on three contexts: 1. the first context; 2. the context that get_basic_info() reopens after its seek test; 3. a third context for audio. Each probe decodes video frames on the calling thread. For a 4K60 H.264 clip at 100 Mbit/s, one probe takes about 100 ms on a 16-core desktop. For common formats, the probe of the audio context is not necessary. The audio context reads the same file as the video context, and MP4/MOV and Matroska/WebM files keep all their stream information in the header. So for these formats, the audio context now gets a copy of what the probe of the video context found. Other formats are still probed, and so is the seek-test context. The copy gives the same result as a probe: the decoded pictures and audio do not change, also after seeks, for MP4, MOV, Matroska and WebM files with AAC, Opus, Vorbis, FLAC, MP3, PCM and AC-3 audio. This is also true for files with several audio streams or with the audio stream first, for fragmented MP4, and for files with an audio offset or an edit list. With this change, a dissolve into a 4K60 H.264 clip at 100 Mbit/s whose file is not open yet goes from 40 to 47 fps. --- src/modules/avformat/producer_avformat.c | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/modules/avformat/producer_avformat.c b/src/modules/avformat/producer_avformat.c index 87552799f0..f7d9a04c00 100644 --- a/src/modules/avformat/producer_avformat.c +++ b/src/modules/avformat/producer_avformat.c @@ -875,6 +875,35 @@ static mlt_image_format pick_image_format(enum AVPixelFormat pix_fmt, return current_format; } +/** Get the stream information of a context that has the same file open as an + * already probed context. + * + * Probing decodes video frames on the calling thread, which pauses playback. MP4/MOV + * and Matroska/WebM files keep all stream information in the file header, so for + * these formats, this copies what the first probe found. Other formats, and files + * where the two contexts have a different number of streams, are probed. + */ +static void find_stream_info_from(AVFormatContext *context, AVFormatContext *probed) +{ + if (!av_match_list(context->iformat->name, "mov,matroska", ',') + || context->nb_streams != probed->nb_streams) { + avformat_find_stream_info(context, NULL); + return; + } + for (unsigned int i = 0; i < context->nb_streams; i++) { + AVStream *stream = context->streams[i]; + AVStream *probed_stream = probed->streams[i]; + avcodec_parameters_copy(stream->codecpar, probed_stream->codecpar); + stream->start_time = probed_stream->start_time; + stream->duration = probed_stream->duration; + stream->avg_frame_rate = probed_stream->avg_frame_rate; + stream->r_frame_rate = probed_stream->r_frame_rate; + } + context->start_time = probed->start_time; + context->duration = probed->duration; + context->bit_rate = probed->bit_rate; +} + static int get_basic_info(producer_avformat self, mlt_profile profile, const char *filename) { int error = 0; @@ -1653,7 +1682,7 @@ static int producer_open( apply_properties(self->audio_format->priv_data, properties, AV_OPT_FLAG_DECODING_PARAM); - avformat_find_stream_info(self->audio_format, NULL); + find_stream_info_from(self->audio_format, self->video_format); } else { self->audio_format = self->video_format; } From cfc6cc28c9540ce49c43cb23127681faa291b647 Mon Sep 17 00:00:00 2001 From: Vinicius Deolindo Date: Tue, 13 Oct 2026 02:11:34 -0300 Subject: [PATCH 5/7] avformat: copy the probed stream information to the seek-test context Previously, when playback reached a clip whose file was not open yet, it also stopped for the probe of the seek-test context. To test if a file is seekable, get_basic_info() seeks in the probed context and keeps it aside. Then it opens the file a second time and probes the new context with avformat_find_stream_info(), which decodes video frames on the calling thread. The audio context already gets its stream information without a probe, from find_stream_info_from(). The reopened context now does the same, so MP4/MOV and Matroska/WebM files skip this probe too. The reopened context has the same stream information as after a probe, and the decoded pictures and audio do not change, also after seeks. This was checked with MP4, MOV, Matroska (H.264 and HEVC), WebM, AVI and album art files. One difference remains, but MLT does not use it: with H.264 and HEVC in Matroska, the packets of the reopened context can have other dts values, because only a probe sets has_b_frames. Their pts values, which MLT uses, are the same. With this change, the start of such a clip takes less time. On a 16-core desktop, a dissolve into a 4K60 H.264 clip at 100 Mbit/s whose file is not open yet goes from 48 to 56 fps. In a Kdenlive project with such clips, the first frame of a dissolve now takes about 260 ms, and it took 290 to 305 ms before. --- src/modules/avformat/producer_avformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/avformat/producer_avformat.c b/src/modules/avformat/producer_avformat.c index f7d9a04c00..2db1aaf875 100644 --- a/src/modules/avformat/producer_avformat.c +++ b/src/modules/avformat/producer_avformat.c @@ -962,7 +962,7 @@ static int get_basic_info(producer_avformat self, mlt_profile profile, const cha self->video_format = self->dummy_context; self->dummy_context = NULL; } else { - avformat_find_stream_info(self->video_format, NULL); + find_stream_info_from(self->video_format, self->dummy_context); } format = self->video_format; } From 4ac17a01326c4ed0298763d1d22eb3c9638a5b12 Mon Sep 17 00:00:00 2001 From: Vinicius Deolindo Date: Tue, 13 Oct 2026 02:13:18 -0300 Subject: [PATCH 6/7] framework: copy large images on several threads Previously, on a 16-core desktop, each frame of a 4K dissolve spent about 3 ms to copy the two decoded frames into the image cache of the avformat producer. The cause was that mlt_frame_clone_image() copied each 16 MB yuv422 image with one memcpy(), and one thread cannot use all of the memory bandwidth. mlt_frame_clone() and mlt_frame_clone_image() now copy large images and alpha masks in slices. The limit is 4 MB, so 4K images use slices, and smaller buffers, such as 1080p yuv422 images, are still copied with one memcpy(). A slice job that starts more slice jobs can deadlock the slice pool. This cannot happen here, because no slice job in MLT clones a frame. To check this, melt ran with several render threads (real_time from -1 to -8) and different slice counts, and it finished normally. The output does not change. With this change, the two copies take about 1.7 ms, and a dissolve between two 4K60 H.264 clips at 100 Mbit/s goes from 104 to 113 fps. --- src/framework/mlt_frame.c | 44 +++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/src/framework/mlt_frame.c b/src/framework/mlt_frame.c index 69680eb405..2129143e3b 100644 --- a/src/framework/mlt_frame.c +++ b/src/framework/mlt_frame.c @@ -26,6 +26,7 @@ #include "mlt_log.h" #include "mlt_producer.h" #include "mlt_profile.h" +#include "mlt_slices.h" #include #include @@ -1334,6 +1335,41 @@ mlt_properties mlt_frame_get_unique_properties(mlt_frame self, mlt_service servi return mlt_properties_get_data(MLT_FRAME_PROPERTIES(self), unique, NULL); } +struct copy_slice_context +{ + uint8_t *dst; + const uint8_t *src; + int size; +}; + +static int copy_slice(int id, int index, int count, void *cookie) +{ + struct copy_slice_context *ctx = cookie; + int start; + int size = mlt_slices_size_slice(count, index, ctx->size, &start); + memcpy(ctx->dst + start, ctx->src + start, size); + return 0; +} + +/** The size from which copy_buffer() uses several threads. Images up to 1080p yuv422 + * (about 4 MB) are copied with one memcpy() as before; 4K images and 1080p rgba + * images are copied in slices. + */ +#define SLICED_COPY_MIN_SIZE (4 << 20) + +/** Copy a buffer, in slices on several threads when it is large, because one thread + * cannot use all of the memory bandwidth. + */ +static void copy_buffer(void *dst, const void *src, int size) +{ + if (size < SLICED_COPY_MIN_SIZE) { + memcpy(dst, src, size); + return; + } + struct copy_slice_context ctx = {.dst = dst, .src = src, .size = size}; + mlt_slices_run_normal(0, copy_slice, &ctx); +} + /** Make a copy of a frame. * * This does not copy the get_image/get_audio processing stacks or any @@ -1387,7 +1423,7 @@ mlt_frame mlt_frame_clone(mlt_frame self, int is_deep) if (!size) size = mlt_image_format_size(format, width, height, NULL); copy = mlt_pool_alloc(size); - memcpy(copy, data, size); + copy_buffer(copy, data, size); mlt_properties_set_data(new_props, "image", copy, size, mlt_pool_release, NULL); size = 0; @@ -1396,7 +1432,7 @@ mlt_frame mlt_frame_clone(mlt_frame self, int is_deep) if (!size) size = width * height; copy = mlt_pool_alloc(size); - memcpy(copy, data, size); + copy_buffer(copy, data, size); mlt_properties_set_data(new_props, "alpha", copy, size, mlt_pool_release, NULL); }; } @@ -1527,7 +1563,7 @@ mlt_frame mlt_frame_clone_image(mlt_frame self, int is_deep) if (!size) size = mlt_image_format_size(format, width, height, NULL); copy = mlt_pool_alloc(size); - memcpy(copy, data, size); + copy_buffer(copy, data, size); mlt_properties_set_data(new_props, "image", copy, size, mlt_pool_release, NULL); size = 0; @@ -1536,7 +1572,7 @@ mlt_frame mlt_frame_clone_image(mlt_frame self, int is_deep) if (!size) size = width * height; copy = mlt_pool_alloc(size); - memcpy(copy, data, size); + copy_buffer(copy, data, size); mlt_properties_set_data(new_props, "alpha", copy, size, mlt_pool_release, NULL); }; } From 25a34c4713842023e3e185ecd3108c56c76c707c Mon Sep 17 00:00:00 2001 From: Vinicius Deolindo Date: Tue, 13 Oct 2026 02:16:50 -0300 Subject: [PATCH 7/7] avformat: find the error concealment frame in the image cache Previously, during forward playback, the producer made a deep copy of each decoded frame in last_good_frame. It needs the last good frame to conceal decoding errors: when a frame fails to decode, the producer shows the last good frame in its place. But the image cache already held a copy of the same frame, so for a 4K clip, each frame cost a second 16 MB copy that was useful only after an error. When the image cache is on, the producer now keeps only the position of the last good frame, and gets the frame from the cache when decoding fails. The cache can lose that frame in two cases, so the producer copies the frame before they happen: - when frames before it go into the cache, which happens during backward playback or after a seek back; - when the cache is reset. Without the image cache, the producer still copies every frame. The producer conceals errors with the same frames as before. This was checked with clips with corrupted parts, played forward, played backward, after a seek back, after a cache reset, and with a small image cache or none. With this change, forward playback is faster. On a 16-core desktop, a dissolve between two 4K60 H.264 clips at 100 Mbit/s goes from 61 to 69 fps, and in a Kdenlive project with such clips, from 73 to 86 fps. --- src/modules/avformat/producer_avformat.c | 71 +++++++++++++++++------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/src/modules/avformat/producer_avformat.c b/src/modules/avformat/producer_avformat.c index 2db1aaf875..d88c88ed61 100644 --- a/src/modules/avformat/producer_avformat.c +++ b/src/modules/avformat/producer_avformat.c @@ -134,6 +134,9 @@ struct producer_avformat_s AVRational video_time_base; mlt_frame last_good_frame; // for video error concealment int last_good_position; // for video error concealment + // For video error concealment: the position of the last good frame in the image + // cache, when last_good_frame is NULL. + mlt_position last_good_cache_position; AVFilterGraph *vfilter_graph; AVFilterContext *vfilter_in; AVFilterContext *vfilter_out; @@ -217,6 +220,7 @@ mlt_producer producer_avformat_init(mlt_profile profile, const char *service, ch if (file) { // Construct the producer producer_avformat self = calloc(1, sizeof(struct producer_avformat_s)); + self->last_good_cache_position = POSITION_INVALID; producer = calloc(1, sizeof(struct mlt_producer_s)); // Initialise it @@ -2866,6 +2870,17 @@ static void init_cache(mlt_properties properties, mlt_cache *cache) mlt_cache_set_size(*cache, cache_size); } +/** Make the last good frame for video error concealment independent of the image + * cache, because the cache is about to lose it. + */ +static void keep_last_good_frame(producer_avformat self) +{ + if (!self->last_good_frame && self->image_cache + && self->last_good_cache_position != POSITION_INVALID) + self->last_good_frame = mlt_cache_get_frame(self->image_cache, + self->last_good_cache_position); +} + /** Get an image from a frame. */ @@ -2912,6 +2927,7 @@ static int producer_get_image(mlt_frame frame, if (self->reset_image_cache) { self->reset_image_cache = 0; + keep_last_good_frame(self); mlt_cache_close(self->image_cache); self->image_cache = NULL; av_frame_free(&self->video_frame); @@ -3351,6 +3367,10 @@ static int producer_get_image(mlt_frame frame, if (image_size > 0) { mlt_properties_set_int(frame_properties, "format", *format); + // A frame before the last good one does not replace it for error concealment, + // but putting it in the image cache can push the last good one out. + if (self->current_position < self->last_good_position) + keep_last_good_frame(self); // Cache the image for rapid repeated access. if (self->image_cache) { if (is_album_art(self)) { @@ -3362,33 +3382,41 @@ static int producer_get_image(mlt_frame frame, mlt_cache_put_frame_image(self->image_cache, frame); } } - // Clone frame for error concealment. + // Keep the frame for error concealment. When the image cache has a copy of it, + // only remember where, so that only a decoding failure needs to copy it. if (self->current_position >= self->last_good_position) { self->last_good_position = self->current_position; if (self->last_good_frame) mlt_frame_close(self->last_good_frame); - self->last_good_frame = mlt_frame_clone(frame, 1); + self->last_good_frame = self->image_cache ? NULL : mlt_frame_clone(frame, 1); + self->last_good_cache_position = self->image_cache ? position : POSITION_INVALID; } - } else if (self->last_good_frame) { + } else { // Use last known good frame if there was a decoding failure. - mlt_frame original = mlt_frame_clone(self->last_good_frame, 1); - mlt_properties orig_props = MLT_FRAME_PROPERTIES(original); - int size = 0; - - *buffer = mlt_frame_get_alpha_size(original, &size); - if (*buffer) - mlt_frame_set_alpha(frame, *buffer, size, NULL); - *buffer = mlt_properties_get_data(orig_props, "image", &size); - mlt_frame_set_image(frame, *buffer, size, NULL); - mlt_properties_set_data(frame_properties, - "avformat.conceal_error", - original, - 0, - (mlt_destructor) mlt_frame_close, - NULL); - *format = mlt_properties_get_int(orig_props, "format"); - set_image_size(self, width, height); - got_picture = 1; + mlt_frame original = NULL; + if (self->last_good_frame) + original = mlt_frame_clone(self->last_good_frame, 1); + else if (self->image_cache && self->last_good_cache_position != POSITION_INVALID) + original = mlt_cache_get_frame(self->image_cache, self->last_good_cache_position); + if (original) { + mlt_properties orig_props = MLT_FRAME_PROPERTIES(original); + int size = 0; + + *buffer = mlt_frame_get_alpha_size(original, &size); + if (*buffer) + mlt_frame_set_alpha(frame, *buffer, size, NULL); + *buffer = mlt_properties_get_data(orig_props, "image", &size); + mlt_frame_set_image(frame, *buffer, size, NULL); + mlt_properties_set_data(frame_properties, + "avformat.conceal_error", + original, + 0, + (mlt_destructor) mlt_frame_close, + NULL); + *format = mlt_properties_get_int(orig_props, "format"); + set_image_size(self, width, height); + got_picture = 1; + } } // Regardless of speed, we expect to get the next frame (cos we ain't too bright) @@ -4575,6 +4603,7 @@ static int producer_get_frame(mlt_producer producer, mlt_frame_ptr frame, int in // If cache miss if (!self) { self = calloc(1, sizeof(struct producer_avformat_s)); + self->last_good_cache_position = POSITION_INVALID; self->parent = producer; mlt_service_cache_put(service, "producer_avformat",