From cd34d3c401ddc2c206d90c0eb87ae53345b44403 Mon Sep 17 00:00:00 2001 From: Brian Matherly Date: Sat, 29 Aug 2026 10:30:39 -0500 Subject: [PATCH 1/6] Add Audio Mixing Assistant --- src/CMakeLists.txt | 1 + src/commands/timelinecommands.cpp | 25 + src/commands/timelinecommands.h | 17 + src/dialogs/audiomixingassistantdialog.cpp | 1186 +++++++++++++++++ src/dialogs/audiomixingassistantdialog.h | 162 +++ src/docks/timelinedock.cpp | 20 + src/docks/timelinedock.h | 1 + src/mainwindow.cpp | 1 + src/models/multitrackmodel.cpp | 89 ++ src/models/multitrackmodel.h | 16 + src/qml/filters/audio_compressor/ui.qml | 6 +- src/qml/filters/audio_eqparametric/ui.qml | 4 +- src/qml/filters/audio_gain/ui.qml | 2 + src/qml/filters/audio_highpass/ui.qml | 4 +- src/qml/filters/audio_lowpass/ui.qml | 4 +- .../Controls/AudioMixingAssistantTag.qml | 42 + src/qml/modules/Shotcut/Controls/qmldir | 1 + src/shotcut_mlt_properties.h | 3 + 18 files changed, 1580 insertions(+), 4 deletions(-) create mode 100644 src/dialogs/audiomixingassistantdialog.cpp create mode 100644 src/dialogs/audiomixingassistantdialog.h create mode 100644 src/qml/modules/Shotcut/Controls/AudioMixingAssistantTag.qml diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3721536597..7ff7280c5e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable(shotcut WIN32 MACOSX_BUNDLE dialogs/addencodepresetdialog.ui dialogs/alignaudiodialog.cpp dialogs/alignaudiodialog.h dialogs/alignmentarray.cpp dialogs/alignmentarray.h + dialogs/audiomixingassistantdialog.cpp dialogs/audiomixingassistantdialog.h dialogs/addonfiltersdialog.cpp dialogs/addonfiltersdialog.h dialogs/addonmetadatahelpdialog.cpp dialogs/addonmetadatahelpdialog.h dialogs/bitratedialog.h dialogs/bitratedialog.cpp diff --git a/src/commands/timelinecommands.cpp b/src/commands/timelinecommands.cpp index 89df9d8fe6..df2a1d15a2 100644 --- a/src/commands/timelinecommands.cpp +++ b/src/commands/timelinecommands.cpp @@ -675,6 +675,31 @@ void NameTrackCommand::undo() m_model.setTrackName(m_trackIndex, m_oldName); } +ChangeTrackRoleCommand::ChangeTrackRoleCommand(MultitrackModel &model, + int trackIndex, + AudioTrackRole role, + QUndoCommand *parent) + : QUndoCommand(parent) + , m_model(model) + , m_trackIndex(qBound(0, trackIndex, qMax(model.rowCount() - 1, 0))) + , m_role(role) + , m_oldRole(model.getTrackRole(trackIndex)) +{ + setText(QObject::tr("Change track role")); +} + +void ChangeTrackRoleCommand::redo() +{ + LOG_DEBUG() << "trackIndex" << m_trackIndex << "role" << m_role; + m_model.setTrackRole(m_trackIndex, m_role); +} + +void ChangeTrackRoleCommand::undo() +{ + LOG_DEBUG() << "trackIndex" << m_trackIndex << "role" << m_oldRole; + m_model.setTrackRole(m_trackIndex, m_oldRole); +} + MergeCommand::MergeCommand(MultitrackModel &model, int trackIndex, int clipIndex, diff --git a/src/commands/timelinecommands.h b/src/commands/timelinecommands.h index dc05aea415..639b99aa24 100644 --- a/src/commands/timelinecommands.h +++ b/src/commands/timelinecommands.h @@ -245,6 +245,23 @@ class NameTrackCommand : public QUndoCommand QString m_oldName; }; +class ChangeTrackRoleCommand : public QUndoCommand +{ +public: + ChangeTrackRoleCommand(MultitrackModel &model, + int trackIndex, + AudioTrackRole role, + QUndoCommand *parent = 0); + void redo(); + void undo(); + +private: + MultitrackModel &m_model; + int m_trackIndex; + AudioTrackRole m_role; + AudioTrackRole m_oldRole; +}; + class MergeCommand : public QUndoCommand { public: diff --git a/src/dialogs/audiomixingassistantdialog.cpp b/src/dialogs/audiomixingassistantdialog.cpp new file mode 100644 index 0000000000..f538182368 --- /dev/null +++ b/src/dialogs/audiomixingassistantdialog.cpp @@ -0,0 +1,1186 @@ +/* + * Copyright (c) 2026 Meltytech, LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "audiomixingassistantdialog.h" + +#include "commands/filtercommands.h" +#include "commands/timelinecommands.h" +#include "controllers/filtercontroller.h" +#include "mainwindow.h" +#include "mltcontroller.h" +#include "models/attachedfiltersmodel.h" +#include "models/multitrackmodel.h" +#include "qmltypes/qmlapplication.h" +#include "shotcut_mlt_properties.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// How far below the program (dialogue anchor) loudness a frame must be to no +// longer count as "dialogue is speaking" for ducking purposes. +static const double kDuckingThresholdOffsetDb = -20.0; +static const int kMinDialogueEventMs = 500; +static const int kMaxDialogueGapMs = 1000; + +// The dialogue compressor's threshold is set above the target program +// loudness so that normal speech passes through untouched. +static const double kCompressorThresholdOffsetDb = 8.0; +static const double kCompressorAttackMs = 15.0; +static const double kCompressorReleaseMs = 200.0; + +// Below this, a track is already close enough to its target loudness that a +// normalization gain filter is not worth recommending. +static const double kGainNormalizationToleranceDb = 0.1; + +static void removeAssistantFilters(Mlt::Producer &producer) +{ + for (int i = producer.filter_count() - 1; i >= 0; --i) { + QScopedPointer filter(producer.filter(i)); + if (filter && filter->is_valid() && filter->get(kShotcutAudioMixingAssistantProperty) + && strlen(filter->get(kShotcutAudioMixingAssistantProperty)) > 0) + producer.detach(*filter); + } +} + +// Measures the integrated loudness of a track, and optionally a frame-by-frame +// level (for tracks that need to be protected, e.g. dialogue), in a worker thread. +class TrackAudioReader : public QObject +{ + Q_OBJECT +public: + TrackAudioReader(int trackIndex, const QString &producerXml, bool measureFrameLevels) + : m_trackIndex(trackIndex) + , m_producerXml(producerXml) + , m_measureFrameLevels(measureFrameLevels) + {} + + void start() { m_future = QtConcurrent::run(&TrackAudioReader::process, this); } + bool isFinished() { return m_future.isFinished(); } + + void process() + { + QScopedPointer producer( + new Mlt::Producer(MLT.profile(), "xml-string", m_producerXml.toUtf8().constData())); + if (!producer->is_valid()) { + emit finished(m_trackIndex, -100.0, QVector()); + return; + } + removeAssistantFilters(*producer); + + Mlt::Filter loudnessFilter(MLT.profile(), "loudness"); + // Without an explicit in/out, mlt_filter_get_length2() falls back to the + // length of whichever individual clip produced the current frame (since + // this filter is attached to the whole track, not a single clip), which + // breaks the filter's end-of-analysis detection. Pin it to the track. + int producerLength = producer->get_length(); + if (producerLength > 0) + loudnessFilter.set_in_and_out(0, producerLength - 1); + producer->attach(loudnessFilter); + + Mlt::Filter channels(MLT.profile(), "audiochannels"); + Mlt::Filter converter(MLT.profile(), "audioconvert"); + Mlt::Filter levelFilter(MLT.profile(), "audiolevel"); + if (m_measureFrameLevels) { + producer->attach(channels); + producer->attach(converter); + producer->attach(levelFilter); + } + + QVector frameLevels; + int n = producer->get_playtime(); + int progress = 0; + for (int i = 0; i < n; i++) { + QScopedPointer frame(producer->get_frame()); + if (frame && frame->is_valid()) { + // Always pull audio, even for silent/blank frames, because the + // loudness filter requires an unbroken sequence of frames. + mlt_audio_format format = mlt_audio_f32le; + int frequency = 48000; + int channelCount = 2; + int samples = mlt_audio_calculate_frame_samples(producer->get_fps(), frequency, i); + frame->get_audio(format, frequency, channelCount, samples); + if (m_measureFrameLevels) { + double left = frame->get_double("meta.media.audio_level.0"); + double right = frame->get_double("meta.media.audio_level.1"); + frameLevels.append(static_cast(qMax(left, right))); + } + } else if (m_measureFrameLevels && !frameLevels.isEmpty()) { + frameLevels.append(frameLevels.back()); + } + + int newProgress = n > 0 ? 100 * (i + 1) / n : 100; + if (newProgress != progress) { + progress = newProgress; + emit progressUpdate(m_trackIndex, progress); + } + } + + double integratedLoudness = -100.0; + QString results = QString::fromUtf8(loudnessFilter.get("results")); + if (!results.isEmpty()) { + QString token = results.split('\t').value(0); // "L: " + int colon = token.indexOf(':'); + if (colon >= 0) + integratedLoudness = token.mid(colon + 1).trimmed().toDouble(); + } + + emit finished(m_trackIndex, integratedLoudness, frameLevels); + } + +signals: + void progressUpdate(int trackIndex, int percent); + void finished(int trackIndex, double integratedLoudness, QVector frameLevels); + +private: + int m_trackIndex; + QString m_producerXml; + bool m_measureFrameLevels; + QFuture m_future; +}; + +AudioMixingAssistantDialog::AudioMixingAssistantDialog(QString title, + MultitrackModel *model, + QWidget *parent) + : QDialog(parent) + , m_model(model) + , m_stage(StageConfigure) + , m_blockUpdates(false) + , m_pendingAnalysisCount(0) +{ + setWindowTitle(title); + setWindowModality(QmlApplication::dialogModality()); + + QVBoxLayout *vlayout = new QVBoxLayout(); + + m_stack = new QStackedWidget(); + m_stack->addWidget(createConfigurePage()); + m_stack->addWidget(createProgressPage()); + m_stack->addWidget(createRecommendationsPage()); + m_stack->setCurrentIndex(StageConfigure); + vlayout->addWidget(m_stack, 1); + + m_buttonBox = new QDialogButtonBox(QDialogButtonBox::Cancel); + connect(m_buttonBox, &QDialogButtonBox::rejected, this, &QDialog::reject); + m_actionButton = m_buttonBox->addButton(tr("Process"), QDialogButtonBox::AcceptRole); + connect(m_actionButton, + &QPushButton::pressed, + this, + &AudioMixingAssistantDialog::onActionButtonPressed); + vlayout->addWidget(m_buttonBox); + + setLayout(vlayout); + setModal(true); + + m_blockUpdates = true; + double targetLoudness = m_model ? m_model->getProjectTargetLoudness() : -14.0; + m_targetLoudnessSpinner->setValue(targetLoudness); + updateTargetPresetFromLoudness(targetLoudness); + m_blockUpdates = false; + + populateTrackTable(); + + resize(600, 400); +} + +QWidget *AudioMixingAssistantDialog::createConfigurePage() +{ + QWidget *page = new QWidget(); + QVBoxLayout *vlayout = new QVBoxLayout(page); + vlayout->setContentsMargins(0, 0, 0, 0); + + QHBoxLayout *targetLayout = new QHBoxLayout(); + targetLayout->addWidget(new QLabel(tr("Target Master Output"))); + m_targetPresetCombo = new QComboBox(); + m_targetPresetCombo->addItem(tr("Web/YouTube (-14 LUFS)")); + m_targetPresetCombo->addItem(tr("Broadcast (-24 LUFS)")); + m_targetPresetCombo->addItem(QString()); + targetLayout->addWidget(m_targetPresetCombo); + m_targetLoudnessSpinner = new QDoubleSpinBox(); + m_targetLoudnessSpinner->setDecimals(1); + m_targetLoudnessSpinner->setRange(-50.0, -5.0); + m_targetLoudnessSpinner->setSingleStep(1.0); + m_targetLoudnessSpinner->setSuffix(tr(" LUFS")); + m_targetLoudnessSpinner->setToolTip(tr("The target integrated loudness of the master output.")); + targetLayout->addWidget(m_targetLoudnessSpinner); + targetLayout->addStretch(1); + vlayout->addLayout(targetLayout); + + QLabel *introLabel = new QLabel( + tr("Assign a role to each track so the assistant knows what to protect and what to " + "level. You can rename tracks here too.")); + introLabel->setWordWrap(true); + vlayout->addWidget(introLabel); + + m_table = new QTableWidget(); + m_table->setColumnCount(3); + m_table->setHorizontalHeaderLabels({tr("Type"), tr("Track Name"), tr("Role")}); + m_table->setSelectionMode(QAbstractItemView::NoSelection); + m_table->setEditTriggers(QAbstractItemView::DoubleClicked | QAbstractItemView::EditKeyPressed); + m_table->verticalHeader()->setVisible(false); + m_table->horizontalHeader()->setSectionResizeMode(ColumnType, QHeaderView::ResizeToContents); + m_table->horizontalHeader()->setSectionResizeMode(ColumnName, QHeaderView::Stretch); + m_table->horizontalHeader()->setSectionResizeMode(ColumnRole, QHeaderView::ResizeToContents); + connect(m_table, + &QTableWidget::itemChanged, + this, + &AudioMixingAssistantDialog::onTrackNameChanged); + vlayout->addWidget(m_table, 1); + + connect(m_targetPresetCombo, + QOverload::of(&QComboBox::currentIndexChanged), + this, + &AudioMixingAssistantDialog::onTargetPresetChanged); + connect(m_targetLoudnessSpinner, + QOverload::of(&QDoubleSpinBox::valueChanged), + this, + &AudioMixingAssistantDialog::onTargetLoudnessChanged); + + return page; +} + +QWidget *AudioMixingAssistantDialog::createProgressPage() +{ + QWidget *page = new QWidget(); + QVBoxLayout *vlayout = new QVBoxLayout(page); + + vlayout->addStretch(1); + QLabel *label = new QLabel(tr("Analyzing tracks...")); + label->setAlignment(Qt::AlignHCenter); + vlayout->addWidget(label); + + m_progressBar = new QProgressBar(); + m_progressBar->setRange(0, 100); + m_progressBar->setValue(0); + vlayout->addWidget(m_progressBar); + vlayout->addStretch(1); + + return page; +} + +QWidget *AudioMixingAssistantDialog::createRecommendationsPage() +{ + QWidget *page = new QWidget(); + QVBoxLayout *vlayout = new QVBoxLayout(page); + vlayout->setContentsMargins(0, 0, 0, 0); + + QLabel *label = new QLabel( + tr("Review the proposed changes below. Uncheck any item you do not want applied.")); + label->setWordWrap(true); + vlayout->addWidget(label); + + m_recommendationsTable = new QTableWidget(); + m_recommendationsTable->setColumnCount(3); + m_recommendationsTable->setHorizontalHeaderLabels( + {tr("Apply"), tr("Track"), tr("Recommendation")}); + m_recommendationsTable->setSelectionMode(QAbstractItemView::NoSelection); + m_recommendationsTable->setEditTriggers(QAbstractItemView::NoEditTriggers); + m_recommendationsTable->setWordWrap(true); + m_recommendationsTable->verticalHeader()->setVisible(false); + m_recommendationsTable->horizontalHeader()->setSectionResizeMode(RecommendationColumnEnabled, + QHeaderView::ResizeToContents); + m_recommendationsTable->horizontalHeader()->setSectionResizeMode(RecommendationColumnTrack, + QHeaderView::ResizeToContents); + m_recommendationsTable->horizontalHeader()->setSectionResizeMode(RecommendationColumnDescription, + QHeaderView::Stretch); + vlayout->addWidget(m_recommendationsTable, 1); + + return page; +} + +QString AudioMixingAssistantDialog::roleDisplayName(AudioTrackRole role) const +{ + switch (role) { + case DialogueAudioRole: + return tr("Dialogue"); + case BackgroundMusicAudioRole: + return tr("Background Music"); + case SoundEffectsAudioRole: + return tr("Sound Effects"); + case AmbienceAudioRole: + return tr("Ambience"); + case PremixedProgramAudioRole: + return tr("Premixed Program"); + case UnassignedAudioRole: + default: + return tr("Unassigned"); + } +} + +void AudioMixingAssistantDialog::populateTrackTable() +{ + if (!m_model) + return; + + m_blockUpdates = true; + m_table->setRowCount(0); + int trackCount = m_model->trackList().size(); + m_table->setRowCount(trackCount); + + for (int row = 0; row < trackCount; row++) { + bool isAudio = m_model->trackList().at(row).type == AudioTrackType; + + auto *typeItem = new QTableWidgetItem(isAudio ? tr("Audio") : tr("Video")); + typeItem->setFlags(typeItem->flags() & ~Qt::ItemIsEditable); + m_table->setItem(row, ColumnType, typeItem); + + auto *nameItem = new QTableWidgetItem(m_model->getTrackName(row)); + nameItem->setData(Qt::UserRole, row); + m_table->setItem(row, ColumnName, nameItem); + + auto *roleCombo = new QComboBox(); + roleCombo->addItem(roleDisplayName(UnassignedAudioRole)); + roleCombo->addItem(roleDisplayName(DialogueAudioRole)); + roleCombo->addItem(roleDisplayName(BackgroundMusicAudioRole)); + roleCombo->addItem(roleDisplayName(SoundEffectsAudioRole)); + roleCombo->addItem(roleDisplayName(AmbienceAudioRole)); + roleCombo->addItem(roleDisplayName(PremixedProgramAudioRole)); + roleCombo->setProperty("trackIndex", row); + roleCombo->setCurrentIndex(static_cast(m_model->getTrackRole(row))); + connect(roleCombo, + QOverload::of(&QComboBox::currentIndexChanged), + this, + [this, row](int index) { onTrackRoleChanged(row, index); }); + m_table->setCellWidget(row, ColumnRole, roleCombo); + } + m_blockUpdates = false; +} + +void AudioMixingAssistantDialog::onTrackNameChanged(QTableWidgetItem *item) +{ + if (m_blockUpdates || !item || item->column() != ColumnName) + return; + int trackIndex = item->data(Qt::UserRole).toInt(); + MAIN.undoStack()->push(new Timeline::NameTrackCommand(*m_model, trackIndex, item->text())); +} + +void AudioMixingAssistantDialog::onTrackRoleChanged(int trackIndex, int roleComboIndex) +{ + if (m_blockUpdates || trackIndex < 0) + return; + MAIN.undoStack()->push( + new Timeline::ChangeTrackRoleCommand(*m_model, + trackIndex, + static_cast(roleComboIndex))); +} + +void AudioMixingAssistantDialog::updateTargetPresetFromLoudness(double value) +{ + bool wasBlocked = m_blockUpdates; + m_blockUpdates = true; + if (qFuzzyCompare(value, -14.0)) + m_targetPresetCombo->setCurrentIndex(PresetWeb); + else if (qFuzzyCompare(value, -24.0)) + m_targetPresetCombo->setCurrentIndex(PresetBroadcast); + else + m_targetPresetCombo->setCurrentIndex(PresetCustom); + m_blockUpdates = wasBlocked; +} + +void AudioMixingAssistantDialog::onTargetPresetChanged(int index) +{ + if (m_blockUpdates) + return; + if (index == PresetWeb) + m_targetLoudnessSpinner->setValue(-14.0); + else if (index == PresetBroadcast) + m_targetLoudnessSpinner->setValue(-24.0); +} + +void AudioMixingAssistantDialog::onTargetLoudnessChanged(double value) +{ + updateTargetPresetFromLoudness(value); + if (m_blockUpdates || !m_model) + return; + m_model->setProjectTargetLoudness(value); +} + +void AudioMixingAssistantDialog::startProcessing() +{ + m_stage = StageProcessing; + m_stack->setCurrentIndex(StageProcessing); + m_actionButton->setEnabled(false); + m_progressBar->setValue(0); + + m_analysisResults.clear(); + m_trackProgressByIndex.clear(); + + QVector readers; + int trackCount = m_model->trackList().size(); + for (int row = 0; row < trackCount; row++) { + AudioTrackRole role = m_model->getTrackRole(row); + if (role == UnassignedAudioRole) + continue; + + int mltIndex = m_model->trackList().at(row).mlt_index; + QScopedPointer track(m_model->tractor()->track(mltIndex)); + if (!track || !track->is_valid()) + continue; + + bool measureFrameLevels = (role == DialogueAudioRole); + auto *reader = new TrackAudioReader(row, MLT.XML(track.data()), measureFrameLevels); + connect(reader, + &TrackAudioReader::progressUpdate, + this, + &AudioMixingAssistantDialog::onTrackAnalysisProgress); + connect(reader, + &TrackAudioReader::finished, + this, + &AudioMixingAssistantDialog::onTrackAnalysisFinished); + m_trackProgressByIndex[row] = 0; + readers.append(reader); + } + + m_pendingAnalysisCount = readers.size(); + if (readers.isEmpty()) { + finishProcessing(); + return; + } + + for (auto *reader : readers) + reader->start(); + + for (auto *reader : readers) { + while (!reader->isFinished()) { + QThread::msleep(10); + QCoreApplication::processEvents(); + } + reader->deleteLater(); + } +} + +void AudioMixingAssistantDialog::onTrackAnalysisProgress(int trackIndex, int percent) +{ + m_trackProgressByIndex[trackIndex] = percent; + int total = 0; + for (int value : std::as_const(m_trackProgressByIndex)) + total += value; + m_progressBar->setValue(total / m_trackProgressByIndex.size()); +} + +void AudioMixingAssistantDialog::onTrackAnalysisFinished(int trackIndex, + double integratedLoudness, + QVector frameLevels) +{ + TrackAnalysisResult result; + result.trackIndex = trackIndex; + result.role = m_model->getTrackRole(trackIndex); + result.integratedLoudness = integratedLoudness; + result.frameLevels = frameLevels; + m_analysisResults.append(result); + + if (--m_pendingAnalysisCount <= 0) + finishProcessing(); +} + +void AudioMixingAssistantDialog::populateRecommendationsList() +{ + m_recommendationsTable->setRowCount(0); + m_recommendationsTable->setRowCount(m_recommendations.size()); + for (int row = 0; row < m_recommendations.size(); row++) { + const auto &recommendation = m_recommendations.at(row); + + auto *enabledItem = new QTableWidgetItem(); + enabledItem->setFlags((enabledItem->flags() & ~Qt::ItemIsEditable) + | Qt::ItemIsUserCheckable); + enabledItem->setCheckState(recommendation.enabled ? Qt::Checked : Qt::Unchecked); + m_recommendationsTable->setItem(row, RecommendationColumnEnabled, enabledItem); + + auto *trackItem = new QTableWidgetItem(m_model->getTrackName(recommendation.trackIndex)); + trackItem->setFlags(trackItem->flags() & ~Qt::ItemIsEditable); + m_recommendationsTable->setItem(row, RecommendationColumnTrack, trackItem); + + auto *descriptionItem = new QTableWidgetItem(recommendation.description); + descriptionItem->setFlags(descriptionItem->flags() & ~Qt::ItemIsEditable); + m_recommendationsTable->setItem(row, RecommendationColumnDescription, descriptionItem); + } + m_recommendationsTable->resizeRowsToContents(); +} + +void AudioMixingAssistantDialog::addLoudnessRecommendation(const TrackAnalysisResult &result) +{ + double targetLoudness = m_model->getProjectTargetLoudness(); + if (std::abs(targetLoudness - result.integratedLoudness) < kGainNormalizationToleranceDb) + return; + double gain = qBound(-70.0, targetLoudness - result.integratedLoudness, 24.0); + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::DialogueLoudnessGain; + recommendation.title = tr("Integrated Loudness Target"); + recommendation.description + = tr("Add Gain/Volume filter: %1 dB (measured %2 LUFS, target %3 LUFS)") + .arg(QString::number(gain, 'f', 1), + QString::number(result.integratedLoudness, 'f', 1), + QString::number(targetLoudness, 'f', 1)); + recommendation.mltService = "volume"; + recommendation.shotcutFilterName = "audioGain"; + recommendation.properties.append({"level", gain}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addHighPassRecommendation(const TrackAnalysisResult &result) +{ + const double cutoffHz = 90.0; // between 80-100 Hz + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::DialogueHighPass; + recommendation.title = tr("High-Pass Filter (Low Cut)"); + recommendation.description = tr("Add High Pass filter at %1 Hz to remove rumble, mic bumps, " + "and wind noise") + .arg(QString::number(cutoffHz, 'f', 0)); + recommendation.mltService = "ladspa.1890"; + recommendation.properties.append({"0", cutoffHz}); // cutoff frequency + recommendation.properties.append({"1", 2.0}); // rolloff rate + recommendation.properties.append({"wetness", 1.0}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addCompressorRecommendation(const TrackAnalysisResult &result) +{ + const double ratio = 2.5; // between 2:1 and 3:1 + const double kneeRadius = 6; // dB, soft knee + double threshold = qBound(-30.0, + m_model->getProjectTargetLoudness() + kCompressorThresholdOffsetDb, + 0.0); + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::DialogueCompressor; + recommendation.title = tr("Dynamic Range Compression"); + recommendation.description = tr("Add Compressor: %1:1 ratio, soft knee, threshold %2 dB, to " + "even out dialogue levels") + .arg(QString::number(ratio, 'f', 1), + QString::number(threshold, 'f', 1)); + recommendation.mltService = "ladspa.1882"; + recommendation.properties.append({"1", kCompressorAttackMs}); + recommendation.properties.append({"2", kCompressorReleaseMs}); + recommendation.properties.append({"3", threshold}); + recommendation.properties.append({"4", ratio}); + recommendation.properties.append({"5", kneeRadius}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::generateRecommendations() +{ + m_recommendations.clear(); + double anchor = dialogueAnchorLoudness(); + for (const auto &result : std::as_const(m_analysisResults)) { + // -100 LUFS (or no measurement at all) means the track has no audible sound. + if (result.integratedLoudness <= -100.0) + continue; + + if (result.role == DialogueAudioRole) { + addLoudnessRecommendation(result); + addHighPassRecommendation(result); + addCompressorRecommendation(result); + } else if (result.role == BackgroundMusicAudioRole) { + addBackgroundMusicGainRecommendation(result, anchor); + addDuckingRecommendation(result); + addFrequencyCarvingRecommendation(result); + } else if (result.role == AmbienceAudioRole) { + addAmbienceGainRecommendation(result, anchor); + addAmbienceHighPassRecommendation(result); + addAmbienceLowPassRecommendation(result); + } else if (result.role == SoundEffectsAudioRole) { + addSfxGainRecommendation(result, anchor); + addSfxHighPassRecommendation(result); + addSfxDuckingRecommendation(result); + } else if (result.role == PremixedProgramAudioRole) { + addPremixedProgramGainRecommendation(result); + } + } +} + +double AudioMixingAssistantDialog::dialogueAnchorLoudness() const +{ + // Dialogue tracks are leveled to the project's target loudness, so that is + // the anchor other roles (e.g. background music) are mixed relative to. + return m_model ? m_model->getProjectTargetLoudness() : -14.0; +} + +void AudioMixingAssistantDialog::addBackgroundMusicGainRecommendation( + const TrackAnalysisResult &result, double anchorLoudness) +{ + const double offsetFromDialogue = -10.0; + double targetLoudness = anchorLoudness + offsetFromDialogue; + if (std::abs(targetLoudness - result.integratedLoudness) < kGainNormalizationToleranceDb) + return; + double gain = qBound(-70.0, targetLoudness - result.integratedLoudness, 24.0); + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::BackgroundMusicGain; + recommendation.title = tr("Integrated Leveling Target (Base Gain)"); + recommendation.description = tr("Add Gain/Volume filter: %1 dB to sit %2 dB below the " + "dialogue anchor (measured %3 LUFS, target %4 LUFS)") + .arg(QString::number(gain, 'f', 1), + QString::number(-offsetFromDialogue, 'f', 0), + QString::number(result.integratedLoudness, 'f', 1), + QString::number(targetLoudness, 'f', 1)); + recommendation.mltService = "volume"; + recommendation.shotcutFilterName = "audioGain"; + recommendation.properties.append({"level", gain}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addDuckingRecommendation(const TrackAnalysisResult &result) +{ + const double attenuationMinDb = -6.0; + const double attenuationMaxDb = -10.0; + const double fadeDownMs = 300.0; // before speech starts + const double fadeUpMs = 500.0; // after speech finishes + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::BackgroundMusicDucking; + recommendation.title = tr("Dynamic Ducking (Speech Overlap)"); + recommendation.description = tr("Add keyframed gain drops of %1 dB to %2 dB while dialogue is " + "active, with a %3 ms fade-down before speech and a %4 ms " + "fade-up after speech") + .arg(QString::number(attenuationMinDb, 'f', 0), + QString::number(attenuationMaxDb, 'f', 0), + QString::number(fadeDownMs, 'f', 0), + QString::number(fadeUpMs, 'f', 0)); + recommendation.mltService = "volume"; + recommendation.shotcutFilterName = "audioGain"; + recommendation.properties.append({"attenuationMinDb", attenuationMinDb}); + recommendation.properties.append({"attenuationMaxDb", attenuationMaxDb}); + recommendation.properties.append({"fadeDownMs", fadeDownMs}); + recommendation.properties.append({"fadeUpMs", fadeUpMs}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addFrequencyCarvingRecommendation(const TrackAnalysisResult &result) +{ + const double dipFrequencyHz = 2000.0; // between 1 kHz and 3 kHz + const double dipGainDb = -3.0; + const double dipQ = 0.7; // broad notch + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::BackgroundMusicFrequencyCarving; + recommendation.title = tr("Frequency Carving (Subtractive EQ)"); + recommendation.description = tr("Add Parametric EQ: %1 dB dip at %2 Hz to carve out space " + "for vocal frequencies") + .arg(QString::number(dipGainDb, 'f', 1), + QString::number(dipFrequencyHz, 'f', 0)); + recommendation.mltService = "ladspa.1204"; + recommendation.shotcutFilterName = "parametricEq"; + // Low shelf and high shelf left flat; only the mid band (props 6-8) dips. + recommendation.properties.append({"0", 0.0}); + recommendation.properties.append({"1", 50.0}); + recommendation.properties.append({"2", 0.5}); + recommendation.properties.append({"3", 0.0}); + recommendation.properties.append({"4", 100.0}); + recommendation.properties.append({"5", 1.0}); + recommendation.properties.append({"6", dipGainDb}); + recommendation.properties.append({"7", dipFrequencyHz}); + recommendation.properties.append({"8", dipQ}); + recommendation.properties.append({"9", 0.0}); + recommendation.properties.append({"10", 5000.0}); + recommendation.properties.append({"11", 1.0}); + recommendation.properties.append({"12", 0.0}); + recommendation.properties.append({"13", 15000.0}); + recommendation.properties.append({"14", 0.5}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addAmbienceGainRecommendation(const TrackAnalysisResult &result, + double anchorLoudness) +{ + const double offsetFromDialogue = -16.0; + double targetLoudness = anchorLoudness + offsetFromDialogue; + if (std::abs(targetLoudness - result.integratedLoudness) < kGainNormalizationToleranceDb) + return; + double gain = qBound(-70.0, targetLoudness - result.integratedLoudness, 24.0); + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::AmbienceGain; + recommendation.title = tr("Integrated Leveling Target (Base Gain)"); + recommendation.description = tr("Add Gain/Volume filter: %1 dB to sit %2 dB below the " + "dialogue anchor (measured %3 LUFS, target %4 LUFS)") + .arg(QString::number(gain, 'f', 1), + QString::number(-offsetFromDialogue, 'f', 0), + QString::number(result.integratedLoudness, 'f', 1), + QString::number(targetLoudness, 'f', 1)); + recommendation.mltService = "volume"; + recommendation.shotcutFilterName = "audioGain"; + recommendation.properties.append({"level", gain}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addAmbienceHighPassRecommendation(const TrackAnalysisResult &result) +{ + const double cutoffHz = 110.0; // between 100-120 Hz + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::AmbienceHighPass; + recommendation.title = tr("Low-Frequency Filtering (High-Pass)"); + recommendation.description = tr("Add High Pass filter at %1 Hz to eliminate HVAC hum, " + "traffic rumble, and wind turbulence") + .arg(QString::number(cutoffHz, 'f', 0)); + recommendation.mltService = "ladspa.1890"; + recommendation.properties.append({"0", cutoffHz}); // cutoff frequency + recommendation.properties.append({"1", 2.0}); // rolloff rate + recommendation.properties.append({"wetness", 1.0}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addAmbienceLowPassRecommendation(const TrackAnalysisResult &result) +{ + const double cutoffHz = 9000.0; // between 8-10 kHz + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::AmbienceLowPass; + recommendation.title = tr("High-Frequency Filtering (Low-Pass)"); + recommendation.description = tr("Add Low Pass filter at %1 Hz to soften hiss, AC whine, and " + "excessive room brightness") + .arg(QString::number(cutoffHz, 'f', 0)); + recommendation.mltService = "ladspa.1891"; + recommendation.properties.append({"0", cutoffHz}); // cutoff frequency + recommendation.properties.append({"1", 2.0}); // rolloff rate + recommendation.properties.append({"wetness", 1.0}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addSfxGainRecommendation(const TrackAnalysisResult &result, + double anchorLoudness) +{ + const double offsetFromDialogue = -6.0; + double targetLoudness = anchorLoudness + offsetFromDialogue; + if (std::abs(targetLoudness - result.integratedLoudness) < kGainNormalizationToleranceDb) + return; + double gain = qBound(-70.0, targetLoudness - result.integratedLoudness, 24.0); + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::SfxGain; + recommendation.title = tr("Integrated Leveling Target (Base Gain)"); + recommendation.description = tr("Add Gain/Volume filter: %1 dB to sit %2 dB below the " + "dialogue/program level (measured %3 LUFS, target %4 LUFS)") + .arg(QString::number(gain, 'f', 1), + QString::number(-offsetFromDialogue, 'f', 0), + QString::number(result.integratedLoudness, 'f', 1), + QString::number(targetLoudness, 'f', 1)); + recommendation.mltService = "volume"; + recommendation.shotcutFilterName = "audioGain"; + recommendation.properties.append({"level", gain}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addSfxHighPassRecommendation(const TrackAnalysisResult &result) +{ + const double cutoffHz = 60.0; + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::SfxHighPass; + recommendation.title = tr("Low-End Management"); + recommendation.description = tr("Add High Pass filter at %1 Hz to prevent low-frequency " + "buildup") + .arg(QString::number(cutoffHz, 'f', 0)); + recommendation.mltService = "ladspa.1890"; + recommendation.properties.append({"0", cutoffHz}); // cutoff frequency + recommendation.properties.append({"1", 2.0}); // rolloff rate + recommendation.properties.append({"wetness", 1.0}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addSfxDuckingRecommendation(const TrackAnalysisResult &result) +{ + const double attenuationDb = -4.0; + const double fadeDownMs = 300.0; // before speech starts + const double fadeUpMs = 500.0; // after speech finishes + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::SfxDucking; + recommendation.title = tr("Dialogue Collision Clearance"); + recommendation.description = tr("Add keyframed gain drops of an additional %1 dB while " + "dialogue is active to maintain clear vocal separation") + .arg(QString::number(attenuationDb, 'f', 0)); + recommendation.mltService = "volume"; + recommendation.shotcutFilterName = "audioGain"; + recommendation.properties.append({"attenuationDb", attenuationDb}); + recommendation.properties.append({"fadeDownMs", fadeDownMs}); + recommendation.properties.append({"fadeUpMs", fadeUpMs}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::addPremixedProgramGainRecommendation( + const TrackAnalysisResult &result) +{ + double targetLoudness = m_model->getProjectTargetLoudness(); + if (std::abs(targetLoudness - result.integratedLoudness) < kGainNormalizationToleranceDb) + return; + double gain = qBound(-70.0, targetLoudness - result.integratedLoudness, 24.0); + + MixRecommendation recommendation; + recommendation.trackIndex = result.trackIndex; + recommendation.kind = RecommendationKind::PremixedProgramGain; + recommendation.title = tr("Integrated Loudness Target"); + recommendation.description = tr("Add Gain/Volume filter: %1 dB to reach the target program " + "loudness (measured %2 LUFS, target %3 LUFS)") + .arg(QString::number(gain, 'f', 1), + QString::number(result.integratedLoudness, 'f', 1), + QString::number(targetLoudness, 'f', 1)); + recommendation.mltService = "volume"; + recommendation.shotcutFilterName = "audioGain"; + recommendation.properties.append({"level", gain}); + m_recommendations.append(recommendation); +} + +void AudioMixingAssistantDialog::finishProcessing() +{ + m_stage = StageRecommendations; + generateRecommendations(); + populateRecommendationsList(); + m_stack->setCurrentIndex(StageRecommendations); + m_recommendationsTable->resizeRowsToContents(); + m_actionButton->setText(tr("Apply")); + m_actionButton->setEnabled(true); +} + +QVector> AudioMixingAssistantDialog::findDialogueEvents() const +{ + // A frame counts as "dialogue active" when any dialogue track's level + // there is within kDuckingThresholdOffsetDb of the program loudness. + double thresholdDb = dialogueAnchorLoudness() + kDuckingThresholdOffsetDb; + + int frameCount = 0; + for (const auto &result : m_analysisResults) + if (result.role == DialogueAudioRole) + frameCount = qMax(frameCount, result.frameLevels.size()); + + QVector active(frameCount, false); + for (const auto &result : m_analysisResults) { + if (result.role != DialogueAudioRole) + continue; + for (int i = 0; i < result.frameLevels.size(); i++) { + double linear = result.frameLevels.at(i); + double db = linear > 0.0 ? 20.0 * std::log10(linear) : -100.0; + if (db >= thresholdDb) + active[i] = true; + } + } + + double fps = MLT.profile().fps(); + int minEventFrames = qRound(kMinDialogueEventMs / 1000.0 * fps); + int maxGapFrames = qRound(kMaxDialogueGapMs / 1000.0 * fps); + + // Find contiguous runs of active frames. + QVector> events; + int start = -1; + for (int i = 0; i < frameCount; i++) { + if (active.at(i) && start < 0) { + start = i; + } else if (!active.at(i) && start >= 0) { + events.append({start, i - 1}); + start = -1; + } + } + if (start >= 0) + events.append({start, frameCount - 1}); + + // Ignore events that are too short to be a real dialogue utterance. + QVector> filtered; + for (const auto &event : std::as_const(events)) { + if (event.second - event.first + 1 >= minEventFrames) + filtered.append(event); + } + + // Combine events that are close enough together to duck as one. + QVector> merged; + for (const auto &event : std::as_const(filtered)) { + if (!merged.isEmpty() && event.first - merged.last().second - 1 < maxGapFrames) + merged.last().second = event.second; + else + merged.append(event); + } + + return merged; +} + +QVector AudioMixingAssistantDialog::generateDuckingKeyframes( + const QVector> &events, + double maxGainDb, + double duckedGainDb, + int fadeDownFrames, + int fadeUpFrames) const +{ + QVector keyframes; + for (const auto &event : events) { + int dialogueStart = event.first; + int dialogueEnd = event.second; + + // 1) Start of the fade-down, still at full volume. + keyframes.append({qMax(dialogueStart - fadeDownFrames, 0), maxGainDb}); + // 2) Minimum (ducked) level, reached by the start of the dialogue. + keyframes.append({dialogueStart, duckedGainDb}); + // 3) Start of the fade-up, at the end of the dialogue. + keyframes.append({dialogueEnd, duckedGainDb}); + // 4) End of the fade-up, back to full volume. + keyframes.append({dialogueEnd + fadeUpFrames, maxGainDb}); + } + return keyframes; +} + +// Looks up a named value from a recommendation's property list. +static double propertyValue(const QList> &properties, + const QString &name, + double defaultValue) +{ + for (const auto &property : properties) { + if (property.first == name) + return property.second; + } + return defaultValue; +} + +static bool isGainKind(RecommendationKind kind) +{ + return kind == RecommendationKind::BackgroundMusicGain || kind == RecommendationKind::SfxGain; +} + +static bool isDuckingKind(RecommendationKind kind) +{ + return kind == RecommendationKind::BackgroundMusicDucking + || kind == RecommendationKind::SfxDucking; +} + +static QString recommendationReason(RecommendationKind kind) +{ + switch (kind) { + case RecommendationKind::DialogueLoudnessGain: + return QObject::tr( + "Gain adjustment to bring dialogue closer to the target program loudness."); + case RecommendationKind::DialogueHighPass: + return QObject::tr( + "High-pass filter to reduce low-frequency rumble, mic bumps, and wind noise."); + case RecommendationKind::DialogueCompressor: + return QObject::tr( + "Compressor to make dialogue volume more consistent and easier to hear."); + case RecommendationKind::BackgroundMusicGain: + return QObject::tr( + "Gain adjustment to keep background music beneath dialogue and other critical audio."); + case RecommendationKind::BackgroundMusicDucking: + return QObject::tr("Ducking to lower background music while dialogue is active."); + case RecommendationKind::BackgroundMusicFrequencyCarving: + return QObject::tr("Parametric EQ to make space for dialogue in the vocal mid-range."); + case RecommendationKind::AmbienceGain: + return QObject::tr("Gain adjustment to keep ambience subtle beneath the dialogue."); + case RecommendationKind::AmbienceHighPass: + return QObject::tr("High-pass filter to reduce HVAC hum, traffic rumble, and wind noise."); + case RecommendationKind::AmbienceLowPass: + return QObject::tr("Low-pass filter to soften hiss, air-conditioner whine, and excessive " + "room brightness."); + case RecommendationKind::SfxGain: + return QObject::tr( + "Gain adjustment to keep sound effects impactful without overpowering dialogue."); + case RecommendationKind::SfxHighPass: + return QObject::tr("High-pass filter to prevent low-frequency buildup in sound effects."); + case RecommendationKind::SfxDucking: + return QObject::tr( + "Ducking to create clearer separation between sound effects and dialogue."); + case RecommendationKind::PremixedProgramGain: + return QObject::tr( + "Gain adjustment to bring the premixed program closer to the target loudness."); + case RecommendationKind::Unknown: + default: + return QString(); + } +} + +void AudioMixingAssistantDialog::applyApprovedRecommendations() +{ + if (!m_model) + return; + + QList rowsToApply; + for (int row = 0; row < m_recommendations.size(); row++) { + if (m_recommendations.at(row).kind == RecommendationKind::Unknown) + continue; + QTableWidgetItem *enabledItem = m_recommendationsTable->item(row, + RecommendationColumnEnabled); + if (enabledItem && enabledItem->checkState() == Qt::Checked) + rowsToApply.append(row); + } + if (rowsToApply.isEmpty()) + return; + + // When both a base gain and a ducking recommendation are approved for the + // same track, they should become one "volume" filter, not two. + QHash gainRowByTrack; + for (int row : std::as_const(rowsToApply)) { + const auto &recommendation = m_recommendations.at(row); + if (isGainKind(recommendation.kind)) + gainRowByTrack[recommendation.trackIndex] = row; + } + QSet consumedRows; + for (int row : std::as_const(rowsToApply)) { + const auto &recommendation = m_recommendations.at(row); + if (isDuckingKind(recommendation.kind) && gainRowByTrack.contains(recommendation.trackIndex)) + consumedRows.insert(gainRowByTrack.value(recommendation.trackIndex)); + } + + AttachedFiltersModel *attachedModel = MAIN.filterController()->attachedModel(); + QSet tracksToReset; + for (int row : std::as_const(rowsToApply)) + tracksToReset.insert(m_recommendations.at(row).trackIndex); + + MAIN.undoStack()->beginMacro(tr("Apply Audio Mixing Recommendations")); + for (int trackIndex : std::as_const(tracksToReset)) { + int mltIndex = m_model->trackList().at(trackIndex).mlt_index; + QScopedPointer track(m_model->tractor()->track(mltIndex)); + if (!track || !track->is_valid()) + continue; + + attachedModel->setProducer(); + attachedModel->setProducer(track.data()); + for (int filterRow = attachedModel->rowCount() - 1; filterRow >= 0; --filterRow) { + QScopedPointer service(attachedModel->getService(filterRow)); + if (service && service->is_valid() && service->get(kShotcutAudioMixingAssistantProperty) + && strlen(service->get(kShotcutAudioMixingAssistantProperty)) > 0) { + MAIN.undoStack()->push(new Filter::RemoveCommand(*attachedModel, + attachedModel->name(filterRow), + *service, + filterRow)); + } + } + } + // Dialogue events are the same for every ducking recommendation, so only compute them once. + QVector> dialogueEvents; + bool haveDialogueEvents = false; + + for (int row : std::as_const(rowsToApply)) { + if (consumedRows.contains(row)) + continue; + const auto &recommendation = m_recommendations.at(row); + int mltIndex = m_model->trackList().at(recommendation.trackIndex).mlt_index; + QScopedPointer track(m_model->tractor()->track(mltIndex)); + if (!track || !track->is_valid()) + continue; + + Mlt::Filter filter(MLT.profile(), recommendation.mltService.toUtf8().constData()); + if (!filter.is_valid()) + continue; + if (!recommendation.shotcutFilterName.isEmpty()) + filter.set(kShotcutFilterProperty, + recommendation.shotcutFilterName.toUtf8().constData()); + filter.set(kShotcutAudioMixingAssistantProperty, + recommendationReason(recommendation.kind).toUtf8().constData()); + filter.set_in_and_out(track->get_in(), track->get_out()); + + QString title = recommendation.title; + if (isDuckingKind(recommendation.kind)) { + if (!haveDialogueEvents) { + dialogueEvents = findDialogueEvents(); + haveDialogueEvents = true; + } + int pairedGainRow = gainRowByTrack.value(recommendation.trackIndex, -1); + double maxGainDb = pairedGainRow >= 0 + ? propertyValue(m_recommendations.at(pairedGainRow).properties, + "level", + 0.0) + : 0.0; + if (pairedGainRow >= 0) + title = tr("%1 + %2").arg(m_recommendations.at(pairedGainRow).title, title); + + double duckedGainDb; + if (recommendation.kind == RecommendationKind::BackgroundMusicDucking) { + double attenuationMinDb = propertyValue(recommendation.properties, + "attenuationMinDb", + -6.0); + double attenuationMaxDb = propertyValue(recommendation.properties, + "attenuationMaxDb", + -10.0); + duckedGainDb = maxGainDb + (attenuationMinDb + attenuationMaxDb) / 2.0; + } else { + duckedGainDb = maxGainDb + + propertyValue(recommendation.properties, "attenuationDb", -4.0); + } + double fps = MLT.profile().fps(); + int fadeDownFrames = qRound( + propertyValue(recommendation.properties, "fadeDownMs", 300.0) / 1000.0 * fps); + int fadeUpFrames = qRound(propertyValue(recommendation.properties, "fadeUpMs", 500.0) + / 1000.0 * fps); + + auto keyframes = generateDuckingKeyframes(dialogueEvents, + maxGainDb, + duckedGainDb, + fadeDownFrames, + fadeUpFrames); + int duration = track->get_length(); + filter.set("level", maxGainDb); + for (const auto &keyframe : std::as_const(keyframes)) { + int clampedFrame = qBound(0, keyframe.frame, qMax(duration - 1, 0)); + filter.anim_set("level", + keyframe.gainDb, + clampedFrame, + duration, + mlt_keyframe_smooth_natural); + } + } else { + for (const auto &property : recommendation.properties) + filter.set(property.first.toUtf8().constData(), property.second); + } + + attachedModel->setProducer(); + attachedModel->setProducer(track.data()); + MAIN.undoStack()->push( + new Filter::AddCommand(*attachedModel, title, filter, attachedModel->rowCount())); + } + MAIN.undoStack()->endMacro(); +} + +void AudioMixingAssistantDialog::onActionButtonPressed() +{ + switch (m_stage) { + case StageConfigure: + startProcessing(); + break; + case StageRecommendations: + applyApprovedRecommendations(); + accept(); + break; + case StageProcessing: + default: + break; + } +} + +#include "audiomixingassistantdialog.moc" diff --git a/src/dialogs/audiomixingassistantdialog.h b/src/dialogs/audiomixingassistantdialog.h new file mode 100644 index 0000000000..c89486edb5 --- /dev/null +++ b/src/dialogs/audiomixingassistantdialog.h @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2026 Meltytech, LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef AUDIOMIXINGASSISTANTDIALOG_H +#define AUDIOMIXINGASSISTANTDIALOG_H + +#include "models/multitrackmodel.h" + +#include +#include +#include + +class QTableWidget; +class QTableWidgetItem; +class QDialogButtonBox; +class QPushButton; +class QComboBox; +class QDoubleSpinBox; +class QStackedWidget; +class QProgressBar; +class TrackAudioReader; + +// The measurements gathered for a single track during analysis. +struct TrackAnalysisResult +{ + int trackIndex = -1; + AudioTrackRole role = UnassignedAudioRole; + double integratedLoudness = -100.0; + QVector frameLevels; // only populated for dialogue tracks +}; + +// A single point in a keyframed gain envelope used to duck a track. +struct DuckingKeyframe +{ + int frame = 0; + double gainDb = 0.0; +}; + +// Identifies what kind of change a recommendation represents, so the Apply +// step knows how to act on it. +enum class RecommendationKind { + Unknown = 0, + DialogueLoudnessGain, + DialogueHighPass, + DialogueCompressor, + BackgroundMusicGain, + BackgroundMusicDucking, + BackgroundMusicFrequencyCarving, + AmbienceGain, + AmbienceHighPass, + AmbienceLowPass, + SfxGain, + SfxHighPass, + SfxDucking, + PremixedProgramGain +}; + +// A single proposed filter to add to a track, pending user approval. +struct MixRecommendation +{ + int trackIndex = -1; + RecommendationKind kind = RecommendationKind::Unknown; + QString title; + QString description; + bool enabled = true; + QString mltService; + QString shotcutFilterName; + QList> properties; +}; + +class AudioMixingAssistantDialog : public QDialog +{ + Q_OBJECT +public: + explicit AudioMixingAssistantDialog(QString title, MultitrackModel *model, QWidget *parent = 0); + +private slots: + void onTrackNameChanged(QTableWidgetItem *item); + void onTrackRoleChanged(int trackIndex, int roleComboIndex); + void onTargetPresetChanged(int index); + void onTargetLoudnessChanged(double value); + void onActionButtonPressed(); + void onTrackAnalysisProgress(int trackIndex, int percent); + void onTrackAnalysisFinished(int trackIndex, + double integratedLoudness, + QVector frameLevels); + +private: + enum Column { ColumnType = 0, ColumnName, ColumnRole }; + enum RecommendationColumn { + RecommendationColumnEnabled = 0, + RecommendationColumnTrack, + RecommendationColumnDescription + }; + enum TargetPreset { PresetWeb = 0, PresetBroadcast, PresetCustom }; + enum Stage { StageConfigure = 0, StageProcessing, StageRecommendations }; + + QWidget *createConfigurePage(); + QWidget *createProgressPage(); + QWidget *createRecommendationsPage(); + void populateTrackTable(); + QString roleDisplayName(AudioTrackRole role) const; + void updateTargetPresetFromLoudness(double value); + void startProcessing(); + void finishProcessing(); + void generateRecommendations(); + void addLoudnessRecommendation(const TrackAnalysisResult &result); + void addHighPassRecommendation(const TrackAnalysisResult &result); + void addCompressorRecommendation(const TrackAnalysisResult &result); + void addBackgroundMusicGainRecommendation(const TrackAnalysisResult &result, + double anchorLoudness); + void addDuckingRecommendation(const TrackAnalysisResult &result); + void addFrequencyCarvingRecommendation(const TrackAnalysisResult &result); + void addAmbienceGainRecommendation(const TrackAnalysisResult &result, double anchorLoudness); + void addAmbienceHighPassRecommendation(const TrackAnalysisResult &result); + void addAmbienceLowPassRecommendation(const TrackAnalysisResult &result); + void addSfxGainRecommendation(const TrackAnalysisResult &result, double anchorLoudness); + void addSfxHighPassRecommendation(const TrackAnalysisResult &result); + void addSfxDuckingRecommendation(const TrackAnalysisResult &result); + void addPremixedProgramGainRecommendation(const TrackAnalysisResult &result); + double dialogueAnchorLoudness() const; + void populateRecommendationsList(); + void applyApprovedRecommendations(); + QVector> findDialogueEvents() const; + QVector generateDuckingKeyframes(const QVector> &events, + double maxGainDb, + double duckedGainDb, + int fadeDownFrames, + int fadeUpFrames) const; + + MultitrackModel *m_model; + QTableWidget *m_table; + QComboBox *m_targetPresetCombo; + QDoubleSpinBox *m_targetLoudnessSpinner; + QStackedWidget *m_stack; + QProgressBar *m_progressBar; + QTableWidget *m_recommendationsTable; + QDialogButtonBox *m_buttonBox; + QPushButton *m_actionButton; + Stage m_stage; + bool m_blockUpdates; + QVector m_analysisResults; + QVector m_recommendations; + QHash m_trackProgressByIndex; + int m_pendingAnalysisCount; +}; + +#endif // AUDIOMIXINGASSISTANTDIALOG_H diff --git a/src/docks/timelinedock.cpp b/src/docks/timelinedock.cpp index d858c80cf9..459dd909f5 100644 --- a/src/docks/timelinedock.cpp +++ b/src/docks/timelinedock.cpp @@ -22,6 +22,7 @@ #include "actions.h" #include "commands/timelinecommands.h" #include "dialogs/alignaudiodialog.h" +#include "dialogs/audiomixingassistantdialog.h" #include "dialogs/durationdialog.h" #include "dialogs/editmarkerdialog.h" #include "dialogs/longuitask.h" @@ -288,6 +289,7 @@ TimelineDock::TimelineDock(QWidget *parent) editMenu->addAction(Actions["timelineRippleTrimClipOutAction"]); editMenu->addAction(Actions["timelineSplitAction"]); editMenu->addAction(Actions["timelineSplitAllTracksAction"]); + editMenu->addAction(Actions["timelineAudioMixingAssistantAction"]); editMenu->addAction(Actions["timelineApplyCopiedFiltersAction"]); editMenu->addAction(Actions["timelineFreezeFrameAction"]); m_mainMenu->addMenu(editMenu); @@ -1685,6 +1687,13 @@ void TimelineDock::setupActions() }); Actions.add("timelineAlignToReferenceAction", action); + action = new QAction(tr("Audio Mixing Assistant..."), this); + connect(action, &QAction::triggered, this, [&]() { + if (m_model.trackList().size() > 1) + showAudioMixingAssistantDialog(); + }); + Actions.add("timelineAudioMixingAssistantAction", action); + action = new QAction(tr("Apply Copied Filters"), this); action->setEnabled(false); connect(action, &QAction::triggered, this, [&]() { @@ -2639,6 +2648,17 @@ void TimelineDock::alignSelectedClips() restoreSelection(); } +/*! + \qmlmethod void TimelineDock::showAudioMixingAssistantDialog() + \brief Shows the Audio Mixing Assistant wizard for leveling and ducking tracks. +*/ + +void TimelineDock::showAudioMixingAssistantDialog() +{ + AudioMixingAssistantDialog dialog(tr("Audio Mixing Assistant"), &m_model, this); + dialog.exec(); +} + void TimelineDock::applyCopiedFiltersToSelectdClips() { QString xmlToUse = QGuiApplication::clipboard()->text(); diff --git a/src/docks/timelinedock.h b/src/docks/timelinedock.h index ddacde7974..5bb1ab4a3b 100644 --- a/src/docks/timelinedock.h +++ b/src/docks/timelinedock.h @@ -136,6 +136,7 @@ public slots: int addAudioTrack(); int addVideoTrack(); void alignSelectedClips(); + void showAudioMixingAssistantDialog(); void onShowFrame(const SharedFrame &frame); void onSeeked(int position); void append(int trackIndex); diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index abeee16d42..d1c35a6735 100644 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -2861,6 +2861,7 @@ void MainWindow::setupActions() addAction(Actions["timelineMergeWithNextAction"]); addAction(Actions["timelineDetachAudioAction"]); addAction(Actions["timelineAlignToReferenceAction"]); + addAction(Actions["timelineAudioMixingAssistantAction"]); addAction(Actions["timelineUpdateThumbnailsAction"]); addAction(Actions["timelineRebuildAudioWaveformAction"]); addAction(Actions["keyframesTypePrevHoldAction"]); diff --git a/src/models/multitrackmodel.cpp b/src/models/multitrackmodel.cpp index dacc26aa7d..8ff0d78dd7 100644 --- a/src/models/multitrackmodel.cpp +++ b/src/models/multitrackmodel.cpp @@ -54,6 +54,40 @@ static QByteArray trackDuckLevelPrefix(int mltTrackIndex) return QStringLiteral("meta.audio.track.%1.").arg(mltTrackIndex).toLatin1(); } +static QString audioRoleToString(AudioTrackRole role) +{ + switch (role) { + case DialogueAudioRole: + return "dialogue"; + case BackgroundMusicAudioRole: + return "background-music"; + case SoundEffectsAudioRole: + return "sound-effects"; + case AmbienceAudioRole: + return "ambience"; + case PremixedProgramAudioRole: + return "premixed-program"; + case UnassignedAudioRole: + default: + return "unassigned"; + } +} + +static AudioTrackRole stringToAudioRole(const QString &str) +{ + if (str == "dialogue") + return DialogueAudioRole; + if (str == "background-music") + return BackgroundMusicAudioRole; + if (str == "sound-effects") + return SoundEffectsAudioRole; + if (str == "ambience") + return AmbienceAudioRole; + if (str == "premixed-program") + return PremixedProgramAudioRole; + return UnassignedAudioRole; +} + static double audioLevelDbFromFilter(Mlt::Filter *filter) { static const char *kAudioLevelLeft = "_audio_level.0"; @@ -443,6 +477,11 @@ QVariant MultitrackModel::data(const QModelIndex &index, int role) const case NameRole: case Qt::DisplayRole: return QString::fromUtf8(track->get(kTrackNameProperty)); + case AudioRoleRole: + return track->property_exists(kTrackAudioRoleProperty) + ? static_cast(stringToAudioRole( + QString::fromUtf8(track->get(kTrackAudioRoleProperty)))) + : static_cast(UnassignedAudioRole); case DurationRole: return playlist.get_playtime(); case IsMuteRole: @@ -583,6 +622,7 @@ QHash MultitrackModel::roleNames() const roles[GroupRole] = "group"; roles[GainRole] = "gain"; roles[GainEnabledRole] = "gainEnabled"; + roles[AudioRoleRole] = "audioRole"; return roles; } @@ -629,6 +669,28 @@ void MultitrackModel::setTrackName(int row, const QString &value) } } +/*! + \qmlmethod void MultitrackModel::setTrackRole(int row, int role) + \brief Sets the audio mixing role of the track at \a row to \a role. +*/ + +void MultitrackModel::setTrackRole(int row, AudioTrackRole role) +{ + if (row < m_trackList.size()) { + int i = m_trackList.at(row).mlt_index; + QScopedPointer track(m_tractor->track(i)); + if (track) { + track->set(kTrackAudioRoleProperty, audioRoleToString(role).toUtf8().constData()); + + QModelIndex modelIndex = index(row, 0); + QVector roles; + roles << AudioRoleRole; + emit dataChanged(modelIndex, modelIndex, roles); + emit modified(); + } + } +} + /*! \qmlmethod void MultitrackModel::setTrackMute(int row, bool mute) \brief Sets the mute state of the track at \a row to \a mute. @@ -3786,6 +3848,33 @@ QString MultitrackModel::getTrackName(int trackIndex) return name; } +AudioTrackRole MultitrackModel::getTrackRole(int trackIndex) +{ + AudioTrackRole role = UnassignedAudioRole; + if (trackIndex < m_trackList.size()) { + QScopedPointer track(m_tractor->track(m_trackList.at(trackIndex).mlt_index)); + if (track && track->property_exists(kTrackAudioRoleProperty)) + role = stringToAudioRole(QString::fromUtf8(track->get(kTrackAudioRoleProperty))); + } + return role; +} + +double MultitrackModel::getProjectTargetLoudness() const +{ + static const double kDefaultTargetLoudness = -14.0; // Web/YouTube preset + if (m_tractor && m_tractor->property_exists(kShotcutProjectTargetLoudnessProperty)) + return m_tractor->get_double(kShotcutProjectTargetLoudnessProperty); + return kDefaultTargetLoudness; +} + +void MultitrackModel::setProjectTargetLoudness(double loudness) +{ + if (m_tractor) { + m_tractor->set(kShotcutProjectTargetLoudnessProperty, loudness); + emit modified(); + } +} + int MultitrackModel::addAudioTrack() { if (!m_tractor) { diff --git a/src/models/multitrackmodel.h b/src/models/multitrackmodel.h index 4693908482..753104e990 100644 --- a/src/models/multitrackmodel.h +++ b/src/models/multitrackmodel.h @@ -37,6 +37,17 @@ typedef enum { VideoTrackType } TrackType; +// The role a track plays in a mix, used by the Audio Mixing Assistant to +// decide how a track should be leveled and whether it should duck others. +typedef enum { + UnassignedAudioRole = 0, + DialogueAudioRole, + BackgroundMusicAudioRole, + SoundEffectsAudioRole, + AmbienceAudioRole, + PremixedProgramAudioRole +} AudioTrackRole; + typedef struct { TrackType type; @@ -93,6 +104,7 @@ class MultitrackModel : public QAbstractItemModel GainRole, /// track, clip GainEnabledRole, /// track, clip IsAdjustmentRole, /// clip only + AudioRoleRole, /// track only }; explicit MultitrackModel(QObject *parent = 0); @@ -143,6 +155,9 @@ class MultitrackModel : public QAbstractItemModel std::unique_ptr getClipInfo(int trackIndex, int clipIndex); Q_INVOKABLE int clipStart(int trackIndex, int clipIndex); QString getTrackName(int trackIndex); + AudioTrackRole getTrackRole(int trackIndex); + double getProjectTargetLoudness() const; + void setProjectTargetLoudness(double loudness); int bottomVideoTrackIndex() const; int mltIndexForTrack(int trackIndex) const; bool checkForEmptyTracks(int trackIndex); @@ -177,6 +192,7 @@ class MultitrackModel : public QAbstractItemModel public slots: void refreshTrackList(); void setTrackName(int row, const QString &value); + void setTrackRole(int row, AudioTrackRole role); void setTrackMute(int row, bool mute); void setTrackHidden(int row, bool hidden); void setTrackComposite(int row, bool composite); diff --git a/src/qml/filters/audio_compressor/ui.qml b/src/qml/filters/audio_compressor/ui.qml index 26320353bf..f7da9aad0b 100644 --- a/src/qml/filters/audio_compressor/ui.qml +++ b/src/qml/filters/audio_compressor/ui.qml @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2025 Meltytech, LLC + * Copyright (c) 2015-2026 Meltytech, LLC * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -65,6 +65,10 @@ Item { anchors.margins: 8 columns: 3 + Shotcut.AudioMixingAssistantTag { + Layout.columnSpan: parent.columns + } + Label { text: qsTr('Preset') Layout.alignment: Qt.AlignRight diff --git a/src/qml/filters/audio_eqparametric/ui.qml b/src/qml/filters/audio_eqparametric/ui.qml index 00cd17f3fb..3c18d3afdf 100644 --- a/src/qml/filters/audio_eqparametric/ui.qml +++ b/src/qml/filters/audio_eqparametric/ui.qml @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2025 Meltytech, LLC + * Copyright (c) 2019-2026 Meltytech, LLC * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -106,6 +106,8 @@ Shotcut.KeyframableFilter { anchors.margins: 8 columns: 4 + Shotcut.AudioMixingAssistantTag {} + Label { text: qsTr('Preset') Layout.alignment: Qt.AlignRight diff --git a/src/qml/filters/audio_gain/ui.qml b/src/qml/filters/audio_gain/ui.qml index 5b7b31bb88..40110c84cc 100644 --- a/src/qml/filters/audio_gain/ui.qml +++ b/src/qml/filters/audio_gain/ui.qml @@ -139,6 +139,8 @@ Item { anchors.fill: parent anchors.margins: 8 + Shotcut.AudioMixingAssistantTag {} + Label { visible: filter.get('shotcut:trackVolume') text: qsTr('%1 Track Volume').arg(filter.get('shotcut:trackVolumeName')) diff --git a/src/qml/filters/audio_highpass/ui.qml b/src/qml/filters/audio_highpass/ui.qml index 9bdf65f969..eab7cd3be9 100644 --- a/src/qml/filters/audio_highpass/ui.qml +++ b/src/qml/filters/audio_highpass/ui.qml @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2025 Meltytech, LLC + * Copyright (c) 2015-2026 Meltytech, LLC * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -70,6 +70,8 @@ Shotcut.KeyframableFilter { anchors.margins: 8 columns: 4 + Shotcut.AudioMixingAssistantTag {} + Label { text: qsTr('Preset') Layout.alignment: Qt.AlignRight diff --git a/src/qml/filters/audio_lowpass/ui.qml b/src/qml/filters/audio_lowpass/ui.qml index c6dd771277..67ad9844f3 100644 --- a/src/qml/filters/audio_lowpass/ui.qml +++ b/src/qml/filters/audio_lowpass/ui.qml @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2025 Meltytech, LLC + * Copyright (c) 2015-2026 Meltytech, LLC * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -70,6 +70,8 @@ Shotcut.KeyframableFilter { anchors.margins: 8 columns: 4 + Shotcut.AudioMixingAssistantTag {} + Label { text: qsTr('Preset') Layout.alignment: Qt.AlignRight diff --git a/src/qml/modules/Shotcut/Controls/AudioMixingAssistantTag.qml b/src/qml/modules/Shotcut/Controls/AudioMixingAssistantTag.qml new file mode 100644 index 0000000000..381e5a16da --- /dev/null +++ b/src/qml/modules/Shotcut/Controls/AudioMixingAssistantTag.qml @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2026 Meltytech, LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import QtQuick +import QtQuick.Controls +import QtQuick.Layouts +import Shotcut.Controls as Shotcut + +Label { + id: root + + property string reason: filter ? filter.get('shotcut:audioMixingAssistant') : '' + + visible: filter && filter.get('shotcut:audioMixingAssistant') !== '' + text: qsTr('Audio Mixing Assistant') + font.bold: true + color: activePalette.text + Layout.columnSpan: 4 + Layout.fillWidth: true + + SystemPalette { + id: activePalette + } + + Shotcut.HoverTip { + text: root.reason + } +} diff --git a/src/qml/modules/Shotcut/Controls/qmldir b/src/qml/modules/Shotcut/Controls/qmldir index 197f4224d6..1da6bda67c 100644 --- a/src/qml/modules/Shotcut/Controls/qmldir +++ b/src/qml/modules/Shotcut/Controls/qmldir @@ -18,6 +18,7 @@ TextFilterVui 1.0 TextFilterVui.qml ToggleButton 1.0 ToggleButton.qml VuiBase 1.0 VuiBase.qml KeyframableFilter 1.0 KeyframableFilter.qml +AudioMixingAssistantTag 1.0 AudioMixingAssistantTag.qml ToolButton 1.0 ToolButton.qml HoverTip 1.0 HoverTip.qml ComboBox 1.0 ComboBox.qml diff --git a/src/shotcut_mlt_properties.h b/src/shotcut_mlt_properties.h index 4e3e0968db..df98227744 100644 --- a/src/shotcut_mlt_properties.h +++ b/src/shotcut_mlt_properties.h @@ -32,6 +32,7 @@ #define kAudioTrackProperty "shotcut:audio" #define kCommentProperty "shotcut:comment" #define kShotcutFilterProperty "shotcut:filter" +#define kShotcutAudioMixingAssistantProperty "shotcut:audioMixingAssistant" #define kShotcutPlaylistProperty "shotcut:playlist" #define kShotcutTransitionProperty "shotcut:transition" #define kShotcutProducerProperty "shotcut:producer" @@ -40,6 +41,7 @@ #define kTrackHeightProperty "shotcut:trackHeight" #define kTrackHeaderWidthProperty "shotcut:trackHeaderWidth" #define kTrackNameProperty "shotcut:name" +#define kTrackAudioRoleProperty "shotcut:audioRole" #define kTrackLockProperty "shotcut:lock" #define kVideoTrackProperty "shotcut:video" #define kShotcutCaptionProperty "shotcut:caption" @@ -74,6 +76,7 @@ #define kShotcutProjectFolder "shotcut:projectFolder" #define kShotcutProjectNote "shotcut:projectNote" #define kShotcutProjectProcessingMode "shotcut:processingMode" +#define kShotcutProjectTargetLoudnessProperty "shotcut:targetLoudness" #define kShotcutColorTransfer "shotcut:colorTransfer" #define kShotcutDuckEnabledProperty "shotcut:duck_enabled" #define kShotcutDuckThresholdProperty "shotcut:duck_threshold" From d176ae9572805558eaea5ee7af42cde83094f4bf Mon Sep 17 00:00:00 2001 From: Brian Matherly Date: Sun, 25 Oct 2026 15:44:18 -0500 Subject: [PATCH 2/6] Cancelling audio mixing analysis no longer waits for the full scan Signal readers to stop between frames and wait for them to exit before closing the dialog, discarding partial results. --- src/dialogs/audiomixingassistantdialog.cpp | 54 ++++++++++++++++++++-- src/dialogs/audiomixingassistantdialog.h | 5 ++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/dialogs/audiomixingassistantdialog.cpp b/src/dialogs/audiomixingassistantdialog.cpp index f538182368..aedd55e6e9 100644 --- a/src/dialogs/audiomixingassistantdialog.cpp +++ b/src/dialogs/audiomixingassistantdialog.cpp @@ -44,6 +44,7 @@ #include #include +#include #include // How far below the program (dialogue anchor) loudness a frame must be to no @@ -86,9 +87,14 @@ class TrackAudioReader : public QObject void start() { m_future = QtConcurrent::run(&TrackAudioReader::process, this); } bool isFinished() { return m_future.isFinished(); } + void cancel() { m_cancelRequested.store(true); } + void waitForFinished() { m_future.waitForFinished(); } void process() { + if (m_cancelRequested.load()) + return; + QScopedPointer producer( new Mlt::Producer(MLT.profile(), "xml-string", m_producerXml.toUtf8().constData())); if (!producer->is_valid()) { @@ -120,6 +126,9 @@ class TrackAudioReader : public QObject int n = producer->get_playtime(); int progress = 0; for (int i = 0; i < n; i++) { + if (m_cancelRequested.load()) + return; + QScopedPointer frame(producer->get_frame()); if (frame && frame->is_valid()) { // Always pull audio, even for silent/blank frames, because the @@ -145,6 +154,9 @@ class TrackAudioReader : public QObject } } + if (m_cancelRequested.load()) + return; + double integratedLoudness = -100.0; QString results = QString::fromUtf8(loudnessFilter.get("results")); if (!results.isEmpty()) { @@ -166,6 +178,7 @@ class TrackAudioReader : public QObject QString m_producerXml; bool m_measureFrameLevels; QFuture m_future; + std::atomic_bool m_cancelRequested{false}; }; AudioMixingAssistantDialog::AudioMixingAssistantDialog(QString title, @@ -190,7 +203,7 @@ AudioMixingAssistantDialog::AudioMixingAssistantDialog(QString title, vlayout->addWidget(m_stack, 1); m_buttonBox = new QDialogButtonBox(QDialogButtonBox::Cancel); - connect(m_buttonBox, &QDialogButtonBox::rejected, this, &QDialog::reject); + connect(m_buttonBox, &QDialogButtonBox::rejected, this, &AudioMixingAssistantDialog::reject); m_actionButton = m_buttonBox->addButton(tr("Process"), QDialogButtonBox::AcceptRole); connect(m_actionButton, &QPushButton::pressed, @@ -424,8 +437,19 @@ void AudioMixingAssistantDialog::onTargetLoudnessChanged(double value) m_model->setProjectTargetLoudness(value); } +void AudioMixingAssistantDialog::reject() +{ + if (m_processing) { + m_cancelRequested = true; + return; + } + QDialog::reject(); +} + void AudioMixingAssistantDialog::startProcessing() { + m_processing = true; + m_cancelRequested = false; m_stage = StageProcessing; m_stack->setCurrentIndex(StageProcessing); m_actionButton->setEnabled(false); @@ -462,6 +486,7 @@ void AudioMixingAssistantDialog::startProcessing() m_pendingAnalysisCount = readers.size(); if (readers.isEmpty()) { + m_processing = false; finishProcessing(); return; } @@ -470,16 +495,36 @@ void AudioMixingAssistantDialog::startProcessing() reader->start(); for (auto *reader : readers) { - while (!reader->isFinished()) { + while (!reader->isFinished() && !m_cancelRequested) { QThread::msleep(10); QCoreApplication::processEvents(); } - reader->deleteLater(); + if (m_cancelRequested) + break; + } + + if (m_cancelRequested) { + for (auto *reader : readers) + reader->cancel(); + } + // Readers must stop using their state before they or the dialog can be destroyed. + for (auto *reader : readers) { + reader->waitForFinished(); + delete reader; + } + m_processing = false; + if (m_cancelRequested) { + m_analysisResults.clear(); + m_pendingAnalysisCount = 0; + QDialog::reject(); } } void AudioMixingAssistantDialog::onTrackAnalysisProgress(int trackIndex, int percent) { + if (m_cancelRequested) + return; + m_trackProgressByIndex[trackIndex] = percent; int total = 0; for (int value : std::as_const(m_trackProgressByIndex)) @@ -491,6 +536,9 @@ void AudioMixingAssistantDialog::onTrackAnalysisFinished(int trackIndex, double integratedLoudness, QVector frameLevels) { + if (m_cancelRequested) + return; + TrackAnalysisResult result; result.trackIndex = trackIndex; result.role = m_model->getTrackRole(trackIndex); diff --git a/src/dialogs/audiomixingassistantdialog.h b/src/dialogs/audiomixingassistantdialog.h index c89486edb5..25e62a3d21 100644 --- a/src/dialogs/audiomixingassistantdialog.h +++ b/src/dialogs/audiomixingassistantdialog.h @@ -88,6 +88,9 @@ class AudioMixingAssistantDialog : public QDialog public: explicit AudioMixingAssistantDialog(QString title, MultitrackModel *model, QWidget *parent = 0); +public slots: + void reject() override; + private slots: void onTrackNameChanged(QTableWidgetItem *item); void onTrackRoleChanged(int trackIndex, int roleComboIndex); @@ -153,6 +156,8 @@ private slots: QPushButton *m_actionButton; Stage m_stage; bool m_blockUpdates; + bool m_processing = false; + bool m_cancelRequested = false; QVector m_analysisResults; QVector m_recommendations; QHash m_trackProgressByIndex; From 4556ef4ef1ac31a256f91110c4c254b17f933e42 Mon Sep 17 00:00:00 2001 From: Brian Matherly Date: Sun, 25 Oct 2026 15:48:42 -0500 Subject: [PATCH 3/6] Do not recommend ducking when there is no dialog Only add background music and sound effects ducking recommendations when dialogue events were detected, so applying them cannot create a gain filter with no ducking keyframes. --- src/dialogs/audiomixingassistantdialog.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/dialogs/audiomixingassistantdialog.cpp b/src/dialogs/audiomixingassistantdialog.cpp index aedd55e6e9..309bc98bd4 100644 --- a/src/dialogs/audiomixingassistantdialog.cpp +++ b/src/dialogs/audiomixingassistantdialog.cpp @@ -643,6 +643,8 @@ void AudioMixingAssistantDialog::generateRecommendations() { m_recommendations.clear(); double anchor = dialogueAnchorLoudness(); + // Ducking without dialogue to duck against would add a filter with no keyframes. + bool hasDialogueEvents = !findDialogueEvents().isEmpty(); for (const auto &result : std::as_const(m_analysisResults)) { // -100 LUFS (or no measurement at all) means the track has no audible sound. if (result.integratedLoudness <= -100.0) @@ -654,7 +656,8 @@ void AudioMixingAssistantDialog::generateRecommendations() addCompressorRecommendation(result); } else if (result.role == BackgroundMusicAudioRole) { addBackgroundMusicGainRecommendation(result, anchor); - addDuckingRecommendation(result); + if (hasDialogueEvents) + addDuckingRecommendation(result); addFrequencyCarvingRecommendation(result); } else if (result.role == AmbienceAudioRole) { addAmbienceGainRecommendation(result, anchor); @@ -663,7 +666,8 @@ void AudioMixingAssistantDialog::generateRecommendations() } else if (result.role == SoundEffectsAudioRole) { addSfxGainRecommendation(result, anchor); addSfxHighPassRecommendation(result); - addSfxDuckingRecommendation(result); + if (hasDialogueEvents) + addSfxDuckingRecommendation(result); } else if (result.role == PremixedProgramAudioRole) { addPremixedProgramGainRecommendation(result); } From c622635e315830e1ee6e7e3b7f6b42be2d09ab07 Mon Sep 17 00:00:00 2001 From: Brian Matherly Date: Sun, 25 Oct 2026 16:07:04 -0500 Subject: [PATCH 4/6] Quiet dialogue no longer misses ducking events Add each dialogue track's planned normalization gain to its frame levels before comparing them against the ducking threshold, sharing the gain calculation with the loudness recommendation. --- src/dialogs/audiomixingassistantdialog.cpp | 20 ++++++++++++++++---- src/dialogs/audiomixingassistantdialog.h | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/dialogs/audiomixingassistantdialog.cpp b/src/dialogs/audiomixingassistantdialog.cpp index 309bc98bd4..510e0efe42 100644 --- a/src/dialogs/audiomixingassistantdialog.cpp +++ b/src/dialogs/audiomixingassistantdialog.cpp @@ -574,12 +574,22 @@ void AudioMixingAssistantDialog::populateRecommendationsList() m_recommendationsTable->resizeRowsToContents(); } +double AudioMixingAssistantDialog::dialogueNormalizationGain(const TrackAnalysisResult &result) const +{ + if (result.integratedLoudness <= -100.0) + return 0.0; + double targetLoudness = dialogueAnchorLoudness(); + if (std::abs(targetLoudness - result.integratedLoudness) < kGainNormalizationToleranceDb) + return 0.0; + return qBound(-70.0, targetLoudness - result.integratedLoudness, 24.0); +} + void AudioMixingAssistantDialog::addLoudnessRecommendation(const TrackAnalysisResult &result) { double targetLoudness = m_model->getProjectTargetLoudness(); - if (std::abs(targetLoudness - result.integratedLoudness) < kGainNormalizationToleranceDb) + double gain = dialogueNormalizationGain(result); + if (gain == 0.0) return; - double gain = qBound(-70.0, targetLoudness - result.integratedLoudness, 24.0); MixRecommendation recommendation; recommendation.trackIndex = result.trackIndex; @@ -930,7 +940,8 @@ void AudioMixingAssistantDialog::finishProcessing() QVector> AudioMixingAssistantDialog::findDialogueEvents() const { // A frame counts as "dialogue active" when any dialogue track's level - // there is within kDuckingThresholdOffsetDb of the program loudness. + // there is within kDuckingThresholdOffsetDb of the program loudness, after + // the planned dialogue normalization gain is applied. double thresholdDb = dialogueAnchorLoudness() + kDuckingThresholdOffsetDb; int frameCount = 0; @@ -942,9 +953,10 @@ QVector> AudioMixingAssistantDialog::findDialogueEvents() const for (const auto &result : m_analysisResults) { if (result.role != DialogueAudioRole) continue; + double gainDb = dialogueNormalizationGain(result); for (int i = 0; i < result.frameLevels.size(); i++) { double linear = result.frameLevels.at(i); - double db = linear > 0.0 ? 20.0 * std::log10(linear) : -100.0; + double db = linear > 0.0 ? 20.0 * std::log10(linear) + gainDb : -100.0; if (db >= thresholdDb) active[i] = true; } diff --git a/src/dialogs/audiomixingassistantdialog.h b/src/dialogs/audiomixingassistantdialog.h index 25e62a3d21..b1cb1a0847 100644 --- a/src/dialogs/audiomixingassistantdialog.h +++ b/src/dialogs/audiomixingassistantdialog.h @@ -136,6 +136,7 @@ private slots: void addSfxDuckingRecommendation(const TrackAnalysisResult &result); void addPremixedProgramGainRecommendation(const TrackAnalysisResult &result); double dialogueAnchorLoudness() const; + double dialogueNormalizationGain(const TrackAnalysisResult &result) const; void populateRecommendationsList(); void applyApprovedRecommendations(); QVector> findDialogueEvents() const; From b24febf2a05371f1cd92ee6e65fe704a0e154f45 Mon Sep 17 00:00:00 2001 From: Brian Matherly Date: Sun, 25 Oct 2026 16:26:26 -0500 Subject: [PATCH 5/6] Duck audio based on dialog or prexmixed tracks Measure frame levels for premixed program tracks and include them with dialogue tracks when detecting the speech events used for ducking, since premixed program audio can also contain dialogue. --- src/dialogs/audiomixingassistantdialog.cpp | 23 ++++++++++++++-------- src/dialogs/audiomixingassistantdialog.h | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/dialogs/audiomixingassistantdialog.cpp b/src/dialogs/audiomixingassistantdialog.cpp index 510e0efe42..c540c7538d 100644 --- a/src/dialogs/audiomixingassistantdialog.cpp +++ b/src/dialogs/audiomixingassistantdialog.cpp @@ -63,6 +63,13 @@ static const double kCompressorReleaseMs = 200.0; // normalization gain filter is not worth recommending. static const double kGainNormalizationToleranceDb = 0.1; +// Premixed program audio may contain dialogue, so it is treated like a +// dialogue track when deciding when to duck other tracks. +static bool isDialogueSourceRole(AudioTrackRole role) +{ + return role == DialogueAudioRole || role == PremixedProgramAudioRole; +} + static void removeAssistantFilters(Mlt::Producer &producer) { for (int i = producer.filter_count() - 1; i >= 0; --i) { @@ -470,7 +477,7 @@ void AudioMixingAssistantDialog::startProcessing() if (!track || !track->is_valid()) continue; - bool measureFrameLevels = (role == DialogueAudioRole); + bool measureFrameLevels = isDialogueSourceRole(role); auto *reader = new TrackAudioReader(row, MLT.XML(track.data()), measureFrameLevels); connect(reader, &TrackAudioReader::progressUpdate, @@ -907,9 +914,9 @@ void AudioMixingAssistantDialog::addPremixedProgramGainRecommendation( const TrackAnalysisResult &result) { double targetLoudness = m_model->getProjectTargetLoudness(); - if (std::abs(targetLoudness - result.integratedLoudness) < kGainNormalizationToleranceDb) + double gain = dialogueNormalizationGain(result); + if (gain == 0.0) return; - double gain = qBound(-70.0, targetLoudness - result.integratedLoudness, 24.0); MixRecommendation recommendation; recommendation.trackIndex = result.trackIndex; @@ -939,19 +946,19 @@ void AudioMixingAssistantDialog::finishProcessing() QVector> AudioMixingAssistantDialog::findDialogueEvents() const { - // A frame counts as "dialogue active" when any dialogue track's level - // there is within kDuckingThresholdOffsetDb of the program loudness, after - // the planned dialogue normalization gain is applied. + // A frame counts as "dialogue active" when any dialogue or premixed program + // track's level there is within kDuckingThresholdOffsetDb of the program + // loudness, after the planned normalization gain is applied. double thresholdDb = dialogueAnchorLoudness() + kDuckingThresholdOffsetDb; int frameCount = 0; for (const auto &result : m_analysisResults) - if (result.role == DialogueAudioRole) + if (isDialogueSourceRole(result.role)) frameCount = qMax(frameCount, result.frameLevels.size()); QVector active(frameCount, false); for (const auto &result : m_analysisResults) { - if (result.role != DialogueAudioRole) + if (!isDialogueSourceRole(result.role)) continue; double gainDb = dialogueNormalizationGain(result); for (int i = 0; i < result.frameLevels.size(); i++) { diff --git a/src/dialogs/audiomixingassistantdialog.h b/src/dialogs/audiomixingassistantdialog.h index b1cb1a0847..9876c1e75b 100644 --- a/src/dialogs/audiomixingassistantdialog.h +++ b/src/dialogs/audiomixingassistantdialog.h @@ -40,7 +40,7 @@ struct TrackAnalysisResult int trackIndex = -1; AudioTrackRole role = UnassignedAudioRole; double integratedLoudness = -100.0; - QVector frameLevels; // only populated for dialogue tracks + QVector frameLevels; // only populated for dialogue and premixed program tracks }; // A single point in a keyframed gain envelope used to duck a track. From d5c8fc1d9a02ed9a65f2f131b470004cb31ff8c8 Mon Sep 17 00:00:00 2001 From: Brian Matherly Date: Sun, 25 Oct 2026 18:40:49 -0500 Subject: [PATCH 6/6] Fix mixing assistant removing filters it did not create. Require the assistant ownership tag before replacing existing filters, so user-added volume, gain, and normalize filters remain untouched. --- src/dialogs/audiomixingassistantdialog.cpp | 55 ++++++++++++---------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/src/dialogs/audiomixingassistantdialog.cpp b/src/dialogs/audiomixingassistantdialog.cpp index c540c7538d..e5731c35e3 100644 --- a/src/dialogs/audiomixingassistantdialog.cpp +++ b/src/dialogs/audiomixingassistantdialog.cpp @@ -70,12 +70,26 @@ static bool isDialogueSourceRole(AudioTrackRole role) return role == DialogueAudioRole || role == PremixedProgramAudioRole; } +static bool isAssistantFilter(Mlt::Properties &properties) +{ + const char *reason = properties.get(kShotcutAudioMixingAssistantProperty); + return reason && *reason; +} + +static bool replacesAssistantFilter(Mlt::Service &service, const MixRecommendation &recommendation) +{ + return isAssistantFilter(service) + && QString::fromUtf8(service.get("mlt_service")) == recommendation.mltService + && (recommendation.shotcutFilterName.isEmpty() + || QString::fromUtf8(service.get(kShotcutFilterProperty)) + == recommendation.shotcutFilterName); +} + static void removeAssistantFilters(Mlt::Producer &producer) { for (int i = producer.filter_count() - 1; i >= 0; --i) { QScopedPointer filter(producer.filter(i)); - if (filter && filter->is_valid() && filter->get(kShotcutAudioMixingAssistantProperty) - && strlen(filter->get(kShotcutAudioMixingAssistantProperty)) > 0) + if (filter && filter->is_valid() && isAssistantFilter(*filter)) producer.detach(*filter); } } @@ -314,7 +328,9 @@ QWidget *AudioMixingAssistantDialog::createRecommendationsPage() vlayout->setContentsMargins(0, 0, 0, 0); QLabel *label = new QLabel( - tr("Review the proposed changes below. Uncheck any item you do not want applied.")); + tr("Review the proposed changes below. Uncheck any item you do not want applied. " + "Existing assistant filters of the same type as checked recommendations " + "will be replaced.")); label->setWordWrap(true); vlayout->addWidget(label); @@ -1129,30 +1145,8 @@ void AudioMixingAssistantDialog::applyApprovedRecommendations() } AttachedFiltersModel *attachedModel = MAIN.filterController()->attachedModel(); - QSet tracksToReset; - for (int row : std::as_const(rowsToApply)) - tracksToReset.insert(m_recommendations.at(row).trackIndex); MAIN.undoStack()->beginMacro(tr("Apply Audio Mixing Recommendations")); - for (int trackIndex : std::as_const(tracksToReset)) { - int mltIndex = m_model->trackList().at(trackIndex).mlt_index; - QScopedPointer track(m_model->tractor()->track(mltIndex)); - if (!track || !track->is_valid()) - continue; - - attachedModel->setProducer(); - attachedModel->setProducer(track.data()); - for (int filterRow = attachedModel->rowCount() - 1; filterRow >= 0; --filterRow) { - QScopedPointer service(attachedModel->getService(filterRow)); - if (service && service->is_valid() && service->get(kShotcutAudioMixingAssistantProperty) - && strlen(service->get(kShotcutAudioMixingAssistantProperty)) > 0) { - MAIN.undoStack()->push(new Filter::RemoveCommand(*attachedModel, - attachedModel->name(filterRow), - *service, - filterRow)); - } - } - } // Dialogue events are the same for every ducking recommendation, so only compute them once. QVector> dialogueEvents; bool haveDialogueEvents = false; @@ -1232,6 +1226,17 @@ void AudioMixingAssistantDialog::applyApprovedRecommendations() attachedModel->setProducer(); attachedModel->setProducer(track.data()); + // Gain and ducking share a Volume filter, so either replaces the whole filter. + for (int filterRow = attachedModel->rowCount() - 1; filterRow >= 0; --filterRow) { + QScopedPointer service(attachedModel->getService(filterRow)); + if (service && service->is_valid() + && replacesAssistantFilter(*service, recommendation)) { + MAIN.undoStack()->push(new Filter::RemoveCommand(*attachedModel, + attachedModel->name(filterRow), + *service, + filterRow)); + } + } MAIN.undoStack()->push( new Filter::AddCommand(*attachedModel, title, filter, attachedModel->rowCount())); }