diff --git a/CMakeLists.txt b/CMakeLists.txt index 63195ff21..6f7b35a72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2032,6 +2032,13 @@ if (ENGINE_BUILD_TESTS) COMMAND audio_chunking_test ) + add_engine_unittest(enhancement_alignment_test tests/unittests/test_enhancement_alignment.cpp) + + add_test( + NAME enhancement_alignment_test + COMMAND enhancement_alignment_test + ) + add_engine_unittest(chinese_normalization_test tests/unittests/test_chinese_normalization.cpp) add_test( diff --git a/include/engine/framework/audio/flashsr.h b/include/engine/framework/audio/flashsr.h index 2b5afa519..9c8d1bac3 100644 --- a/include/engine/framework/audio/flashsr.h +++ b/include/engine/framework/audio/flashsr.h @@ -17,6 +17,28 @@ struct FlashSrOutput { std::vector samples; }; +// Absolute peak the FlashSR output is never allowed to exceed. +inline constexpr float kFlashSrPeakCeiling = 0.9990000128746033f; + +struct FlashSrOptions { + // Restore the input waveform's peak on the output instead of normalising + // every file to `peak_ceiling`. With this off a single loud sample sets the + // level of the whole file and the input's own level is discarded — a -40 dBFS + // whisper and a -3 dBFS shout both come back at -0.009 dBFS. Set false for + // bit-exact parity with the upstream reference implementation, which is what + // the reference fixtures under tests/unittests/assets were captured with. + bool preserve_input_level = true; + // Safety limit, not a target. The output is scaled down only if restoring + // the input peak would push it above this value. + float peak_ceiling = kFlashSrPeakCeiling; +}; + +// Gain applied to a FlashSR output whose absolute peak is `output_peak`, given +// an input whose absolute peak is `input_peak`. Returns 0 for a silent output +// (silence in, silence out) and never lets the result exceed +// `options.peak_ceiling`. +float flashsr_output_gain(float input_peak, float output_peak, const FlashSrOptions & options) noexcept; + class FlashSrModel { public: static FlashSrModel load_from_directory(const std::filesystem::path & model_dir); @@ -29,7 +51,9 @@ class FlashSrModel { FlashSrModel(const FlashSrModel &) = delete; FlashSrModel & operator=(const FlashSrModel &) = delete; - FlashSrOutput super_resolve_mono_16k(const std::vector & waveform) const; + FlashSrOutput super_resolve_mono_16k( + const std::vector & waveform, + const FlashSrOptions & options = FlashSrOptions{}) const; private: explicit FlashSrModel(std::shared_ptr weights); diff --git a/include/engine/framework/audio/rnnoise.h b/include/engine/framework/audio/rnnoise.h index 8e2ebaadb..5fe5d9db2 100644 --- a/include/engine/framework/audio/rnnoise.h +++ b/include/engine/framework/audio/rnnoise.h @@ -38,6 +38,34 @@ struct RnnoiseWaveformOutput { std::vector vad; }; +// Group delay of the RNNoise analysis/synthesis pipeline, in samples at 48 kHz. +// The synthesis stage inverse-transforms the *previous* frame's spectrum, so an +// output sample at index n reconstructs the input sample at index n - 960 +// (20 ms). DeepFilterNet2 crops its own 480-sample delay the same way. +inline constexpr int64_t kRnnoiseOutputDelaySamples = 960; + +struct RnnoiseProcessOptions { + // Pad the input tail by kRnnoiseOutputDelaySamples and crop the same amount + // off the front of the synthesis output, so that output[n] lines up with + // input[n] and the last 20 ms of input still reaches the output. Set false + // for bit-exact parity with the upstream reference implementation, which + // leaves the delay in — that is what the reference fixtures under + // tests/unittests/assets were captured with. + bool compensate_output_delay = true; +}; + +// Frame, padding and crop arithmetic process_mono_48k uses for a given input +// length. Exposed so the alignment can be exercised without model weights. +struct RnnoiseAlignmentPlan { + int64_t frames = 0; + int64_t padded_samples = 0; + int64_t crop_offset = 0; + int64_t output_samples = 0; + int64_t vad_frames = 0; +}; + +RnnoiseAlignmentPlan rnnoise_alignment_plan(int64_t input_samples, const RnnoiseProcessOptions & options) noexcept; + class RnnoiseModel { public: static RnnoiseModel load_from_safetensors(const std::filesystem::path & checkpoint_path); @@ -57,7 +85,9 @@ class RnnoiseModel { const std::vector & features, int64_t frames, int64_t feature_size) const; - RnnoiseWaveformOutput process_mono_48k(const std::vector & waveform) const; + RnnoiseWaveformOutput process_mono_48k( + const std::vector & waveform, + const RnnoiseProcessOptions & options = RnnoiseProcessOptions{}) const; std::unique_ptr create_streaming_session() const; diff --git a/include/engine/framework/audio/zipenhancer.h b/include/engine/framework/audio/zipenhancer.h index 33373ba18..d8b7360c5 100644 --- a/include/engine/framework/audio/zipenhancer.h +++ b/include/engine/framework/audio/zipenhancer.h @@ -2,6 +2,7 @@ #include "engine/framework/core/backend.h" +#include #include #include #include @@ -15,6 +16,51 @@ struct ZipEnhancerWaveformOutput { std::vector samples; }; +// Chunk-join and output-length policy for ZipEnhancerModel::denoise_mono_16k. +struct ZipEnhancerOptions { + // Length in samples (at 16 kHz) of the linear crossfade applied where two + // consecutive 2 s analysis windows meet. Clamped to the 8000-sample (500 ms) + // overlap between windows. 0 restores the legacy hard splice, where the + // first half of the overlap comes from the earlier chunk and the second half + // from the later one with no fade between them. + int64_t chunk_crossfade_samples = 8000; + // When true denoise_mono_16k returns exactly as many samples as it was + // given. When false the un-segmented path returns floor(n / 100) * 100 + // samples, which is what the upstream reference implementation emits and + // what the reference fixtures under tests/unittests/assets were captured + // with. + bool match_input_length = true; +}; + +// Segmentation geometry denoise_mono_16k uses for a given input length. Exposed +// so the length and coverage arithmetic can be exercised without model weights. +struct ZipEnhancerChunkPlan { + int64_t window_samples = 0; + int64_t stride_samples = 0; + int64_t padded_samples = 0; + int64_t output_samples = 0; + bool segmented = false; +}; + +ZipEnhancerChunkPlan zipenhancer_chunk_plan(int64_t input_samples, const ZipEnhancerOptions & options) noexcept; + +// Rising crossfade weight at `position` inside an `overlap_samples`-long chunk +// join, using a linear ramp of `fade_samples` centred in the overlap. The pair +// (position, overlap_samples - 1 - position) always sums to exactly 1. +float zipenhancer_chunk_fade_weight(int64_t position, int64_t overlap_samples, int64_t fade_samples) noexcept; + +// Weight the chunk starting at `segment_start` contributes to the output sample +// at `segment_start + offset_in_segment`. +float zipenhancer_segment_weight( + int64_t offset_in_segment, + int64_t segment_start, + const ZipEnhancerChunkPlan & plan, + const ZipEnhancerOptions & options) noexcept; + +// Total overlap-add weight every padded output sample receives. Every entry +// below plan.output_samples must be strictly positive or the join leaves a hole. +std::vector zipenhancer_chunk_weights(const ZipEnhancerChunkPlan & plan, const ZipEnhancerOptions & options); + class ZipEnhancerModel { public: static ZipEnhancerModel load_from_directory(const std::filesystem::path & model_dir); @@ -27,7 +73,9 @@ class ZipEnhancerModel { ZipEnhancerModel(const ZipEnhancerModel &) = delete; ZipEnhancerModel & operator=(const ZipEnhancerModel &) = delete; - ZipEnhancerWaveformOutput denoise_mono_16k(const std::vector & waveform) const; + ZipEnhancerWaveformOutput denoise_mono_16k( + const std::vector & waveform, + const ZipEnhancerOptions & options = ZipEnhancerOptions{}) const; private: explicit ZipEnhancerModel(std::shared_ptr state); diff --git a/src/framework/audio/flashsr.cpp b/src/framework/audio/flashsr.cpp index 7c959691d..e95e5b4ff 100644 --- a/src/framework/audio/flashsr.cpp +++ b/src/framework/audio/flashsr.cpp @@ -29,7 +29,6 @@ constexpr int kFlashSrOutputSampleRate = 48000; constexpr int kFlashSrChannels = 32; constexpr int kFlashSrActivationKernel = 12; constexpr int kFlashSrActivationRatio = 2; -constexpr float kFlashSrOutputScale = 0.9990000128746033f; struct GgmlContextDeleter { void operator()(ggml_context * ctx) const noexcept { @@ -302,22 +301,32 @@ core::TensorValue resblock( return output; } -std::vector normalize_output(const std::vector & input) { - float max_abs = 0.0f; - for (float value : input) { - max_abs = std::max(max_abs, std::fabs(value)); - } - if (max_abs <= 0.0f) { - throw std::runtime_error("FlashSR output normalization has zero peak"); +namespace { + +float absolute_peak(const std::vector & values) noexcept { + float peak = 0.0f; + for (const float value : values) { + peak = std::max(peak, std::fabs(value)); } - std::vector output(input.size()); - const float scale = kFlashSrOutputScale / max_abs; - for (size_t i = 0; i < input.size(); ++i) { - output[i] = input[i] * scale; + return peak; +} + +// Level policy for the model output. `source` is the waveform the caller handed +// in, so the input's own level can be restored instead of being discarded. +std::vector apply_output_level( + const std::vector & source, + const std::vector & model_output, + const FlashSrOptions & options) { + const float gain = flashsr_output_gain(absolute_peak(source), absolute_peak(model_output), options); + std::vector output(model_output.size()); + for (size_t i = 0; i < model_output.size(); ++i) { + output[i] = model_output[i] * gain; } return output; } +} // namespace + class FlashSrGraph { public: FlashSrGraph(const FlashSrWeights & weights, int64_t input_samples) @@ -402,6 +411,17 @@ class FlashSrGraph { ggml_backend_graph_plan_t plan_ = nullptr; }; +float flashsr_output_gain(float input_peak, float output_peak, const FlashSrOptions & options) noexcept { + if (!(output_peak > 0.0f)) { + return 0.0f; + } + const float ceiling = options.peak_ceiling > 0.0f ? options.peak_ceiling : kFlashSrPeakCeiling; + const float target = options.preserve_input_level + ? std::min(std::max(input_peak, 0.0f), ceiling) + : ceiling; + return target / output_peak; +} + FlashSrModel::FlashSrModel() = default; FlashSrModel::~FlashSrModel() = default; FlashSrModel::FlashSrModel(FlashSrModel &&) noexcept = default; @@ -447,7 +467,9 @@ FlashSrModel FlashSrModel::load_from_directory( return FlashSrModel(std::move(weights)); } -FlashSrOutput FlashSrModel::super_resolve_mono_16k(const std::vector & waveform) const { +FlashSrOutput FlashSrModel::super_resolve_mono_16k( + const std::vector & waveform, + const FlashSrOptions & options) const { if (!weights_) { throw std::runtime_error("FlashSR model is not loaded"); } @@ -463,7 +485,9 @@ FlashSrOutput FlashSrModel::super_resolve_mono_16k(const std::vector & wa if (!graph_ || !graph_->matches(original_samples)) { graph_ = std::make_unique(*weights_, original_samples); } - return FlashSrOutput{kFlashSrOutputSampleRate, normalize_output(graph_->run(waveform))}; + return FlashSrOutput{ + kFlashSrOutputSampleRate, + apply_output_level(waveform, graph_->run(waveform), options)}; } std::vector padded = waveform; @@ -510,7 +534,7 @@ FlashSrOutput FlashSrModel::super_resolve_mono_16k(const std::vector & wa } output[i] /= weights[i]; } - return FlashSrOutput{kFlashSrOutputSampleRate, normalize_output(output)}; + return FlashSrOutput{kFlashSrOutputSampleRate, apply_output_level(waveform, output, options)}; } } // namespace engine::audio diff --git a/src/framework/audio/rnnoise.cpp b/src/framework/audio/rnnoise.cpp index 204f4fe9e..367484be4 100644 --- a/src/framework/audio/rnnoise.cpp +++ b/src/framework/audio/rnnoise.cpp @@ -1402,12 +1402,31 @@ RnnoiseSequenceOutput RnnoiseModel::infer_features( return output; } -RnnoiseWaveformOutput RnnoiseModel::process_mono_48k(const std::vector & waveform) const { +RnnoiseAlignmentPlan rnnoise_alignment_plan(int64_t input_samples, const RnnoiseProcessOptions & options) noexcept { + static_assert( + kRnnoiseOutputDelaySamples == 2 * static_cast(kRnnoiseFrameSize), + "RNNoise group delay must stay two analysis frames"); + RnnoiseAlignmentPlan plan; + if (input_samples <= 0) { + return plan; + } + plan.crop_offset = options.compensate_output_delay ? kRnnoiseOutputDelaySamples : 0; + plan.output_samples = input_samples; + plan.vad_frames = (input_samples + kRnnoiseFrameSize - 1) / kRnnoiseFrameSize; + plan.frames = (input_samples + plan.crop_offset + kRnnoiseFrameSize - 1) / kRnnoiseFrameSize; + plan.padded_samples = plan.frames * kRnnoiseFrameSize; + return plan; +} + +RnnoiseWaveformOutput RnnoiseModel::process_mono_48k( + const std::vector & waveform, + const RnnoiseProcessOptions & options) const { if (waveform.empty()) { throw std::runtime_error("RNNoise waveform input is empty"); } - const int64_t frames = (static_cast(waveform.size()) + kRnnoiseFrameSize - 1) / kRnnoiseFrameSize; - std::vector padded(static_cast(frames * kRnnoiseFrameSize), 0.0f); + const auto plan = rnnoise_alignment_plan(static_cast(waveform.size()), options); + const int64_t frames = plan.frames; + std::vector padded(static_cast(plan.padded_samples), 0.0f); std::copy(waveform.begin(), waveform.end(), padded.begin()); std::vector output(padded.size(), 0.0f); std::vector vad; @@ -1448,8 +1467,16 @@ RnnoiseWaveformOutput RnnoiseModel::process_mono_48k(const std::vector & } offset += chunk_frames; } - output.resize(waveform.size()); - return RnnoiseWaveformOutput{kRnnoiseSampleRate, std::move(output), std::move(vad)}; + if (static_cast(output.size()) < plan.crop_offset + plan.output_samples) { + throw std::runtime_error("RNNoise synthesis output is shorter than the delay-compensated crop"); + } + std::vector aligned( + output.begin() + static_cast(plan.crop_offset), + output.begin() + static_cast(plan.crop_offset + plan.output_samples)); + if (static_cast(vad.size()) > plan.vad_frames) { + vad.resize(static_cast(plan.vad_frames)); + } + return RnnoiseWaveformOutput{kRnnoiseSampleRate, std::move(aligned), std::move(vad)}; } RnnoiseStreamingSession::RnnoiseStreamingSession(std::shared_ptr weights) diff --git a/src/framework/audio/zipenhancer.cpp b/src/framework/audio/zipenhancer.cpp index 7a05a497c..cdbb02771 100644 --- a/src/framework/audio/zipenhancer.cpp +++ b/src/framework/audio/zipenhancer.cpp @@ -36,6 +36,9 @@ constexpr int64_t kNfft = 400; constexpr int64_t kHop = 100; constexpr int64_t kWin = 400; constexpr int64_t kFreqBins = 201; +constexpr int64_t kWindowSamples = kSampleRate * 2; // 32000, 2 s +constexpr int64_t kStrideSamples = kWindowSamples * 3 / 4; // 24000, 1.5 s +constexpr int64_t kSegmentThreshold = kWindowSamples * 3; // 96000, 6 s constexpr int64_t kDense = 64; constexpr int64_t kHeads = 4; constexpr int64_t kQueryHeadDim = 12; @@ -1045,9 +1048,81 @@ struct ZipEnhancerModelState { mutable std::unique_ptr forward_graph; }; +ZipEnhancerChunkPlan zipenhancer_chunk_plan(int64_t input_samples, const ZipEnhancerOptions & options) noexcept { + ZipEnhancerChunkPlan plan; + plan.window_samples = kWindowSamples; + plan.stride_samples = kStrideSamples; + if (input_samples <= 0) { + return plan; + } + if (input_samples <= kSegmentThreshold) { + plan.padded_samples = std::max(input_samples, kWindowSamples); + plan.output_samples = (options.match_input_length || input_samples < kWindowSamples) + ? input_samples + : (input_samples / kHop) * kHop; + return plan; + } + plan.segmented = true; + const int64_t remainder = (input_samples - kWindowSamples) % kStrideSamples; + plan.padded_samples = input_samples + (remainder == 0 ? 0 : kStrideSamples - remainder); + plan.output_samples = input_samples; + return plan; +} + +float zipenhancer_chunk_fade_weight(int64_t position, int64_t overlap_samples, int64_t fade_samples) noexcept { + if (overlap_samples <= 0) { + return 1.0f; + } + const int64_t fade = std::clamp(fade_samples, 0, overlap_samples); + const int64_t lead = (overlap_samples - fade) / 2; + if (position < lead) { + return 0.0f; + } + if (position >= lead + fade) { + return 1.0f; + } + return static_cast(position - lead + 1) / static_cast(fade + 1); +} + +float zipenhancer_segment_weight( + int64_t offset_in_segment, + int64_t segment_start, + const ZipEnhancerChunkPlan & plan, + const ZipEnhancerOptions & options) noexcept { + const int64_t overlap = plan.window_samples - plan.stride_samples; + float weight = 1.0f; + if (segment_start > 0 && offset_in_segment < overlap) { + weight = zipenhancer_chunk_fade_weight(offset_in_segment, overlap, options.chunk_crossfade_samples); + } + if (segment_start + plan.window_samples < plan.padded_samples && offset_in_segment >= plan.stride_samples) { + weight = zipenhancer_chunk_fade_weight( + overlap - 1 - (offset_in_segment - plan.stride_samples), + overlap, + options.chunk_crossfade_samples); + } + return weight; +} + +std::vector zipenhancer_chunk_weights(const ZipEnhancerChunkPlan & plan, const ZipEnhancerOptions & options) { + if (plan.padded_samples <= 0) { + return {}; + } + if (!plan.segmented) { + return std::vector(static_cast(plan.padded_samples), 1.0f); + } + std::vector weights(static_cast(plan.padded_samples), 0.0f); + for (int64_t current = 0; current + plan.window_samples <= plan.padded_samples; current += plan.stride_samples) { + for (int64_t i = 0; i < plan.window_samples; ++i) { + weights[static_cast(current + i)] += zipenhancer_segment_weight(i, current, plan, options); + } + } + return weights; +} + ZipEnhancerWaveformOutput denoise_mono_16k_whole( const ZipEnhancerModelState & state, - const std::vector & waveform) { + const std::vector & waveform, + int64_t requested_samples) { if (waveform.empty()) { throw std::runtime_error("ZipEnhancer input waveform is empty"); } @@ -1107,7 +1182,7 @@ ZipEnhancerWaveformOutput denoise_mono_16k_whole( out_complex[idx * 2 + 1] = linear_mag * std::sin(phase); } } - const int64_t output_samples = (frames - 1) * kHop; + const int64_t output_samples = requested_samples > 0 ? requested_samples : (frames - 1) * kHop; auto wav = ISTFT().compute( out_complex, window, @@ -1169,7 +1244,9 @@ ZipEnhancerModel ZipEnhancerModel::load_from_directory( return ZipEnhancerModel(std::move(state)); } -ZipEnhancerWaveformOutput ZipEnhancerModel::denoise_mono_16k(const std::vector & waveform) const { +ZipEnhancerWaveformOutput ZipEnhancerModel::denoise_mono_16k( + const std::vector & waveform, + const ZipEnhancerOptions & options) const { if (!state_) { throw std::runtime_error("ZipEnhancerModel is not loaded"); } @@ -1177,50 +1254,43 @@ ZipEnhancerWaveformOutput ZipEnhancerModel::denoise_mono_16k(const std::vector(waveform.size()); - if (original_samples < window_samples) { - std::vector padded = waveform; - padded.resize(static_cast(window_samples), 0.0f); - auto output = denoise_mono_16k_whole(*state_, padded); - output.samples.resize(static_cast(original_samples)); - return output; - } - if (original_samples <= segment_threshold) { - return denoise_mono_16k_whole(*state_, waveform); + const auto plan = zipenhancer_chunk_plan(original_samples, options); + if (!plan.segmented) { + if (original_samples < plan.window_samples) { + std::vector padded = waveform; + padded.resize(static_cast(plan.window_samples), 0.0f); + auto output = denoise_mono_16k_whole(*state_, padded, plan.window_samples); + output.samples.resize(static_cast(plan.output_samples)); + return output; + } + return denoise_mono_16k_whole(*state_, waveform, plan.output_samples); } std::vector padded = waveform; - const int64_t remainder = (original_samples - window_samples) % stride_samples; - if (remainder != 0) { - padded.insert(padded.end(), static_cast(stride_samples - remainder), 0.0f); - } - const int64_t padded_samples = static_cast(padded.size()); - std::vector output(static_cast(padded_samples), 0.0f); - for (int64_t current = 0; current + window_samples <= padded_samples; current += stride_samples) { + padded.resize(static_cast(plan.padded_samples), 0.0f); + std::vector output(static_cast(plan.padded_samples), 0.0f); + const auto weights = zipenhancer_chunk_weights(plan, options); + for (int64_t current = 0; current + plan.window_samples <= plan.padded_samples; current += plan.stride_samples) { std::vector segment( padded.begin() + static_cast(current), - padded.begin() + static_cast(current + window_samples)); - auto segment_output = denoise_mono_16k_whole(*state_, segment); - if (static_cast(segment_output.samples.size()) < window_samples) { - throw std::runtime_error("ZipEnhancer segmented output is shorter than its input window"); + padded.begin() + static_cast(current + plan.window_samples)); + const auto segment_output = denoise_mono_16k_whole(*state_, segment, plan.window_samples); + if (static_cast(segment_output.samples.size()) != plan.window_samples) { + throw std::runtime_error("ZipEnhancer segmented output length mismatch"); + } + for (int64_t i = 0; i < plan.window_samples; ++i) { + const float weight = zipenhancer_segment_weight(i, current, plan, options); + output[static_cast(current + i)] += segment_output.samples[static_cast(i)] * weight; } - if (current == 0) { - std::copy( - segment_output.samples.begin(), - segment_output.samples.begin() + static_cast(window_samples - give_up_samples), - output.begin()); - } else { - std::copy( - segment_output.samples.begin() + static_cast(give_up_samples), - segment_output.samples.begin() + static_cast(window_samples - give_up_samples), - output.begin() + static_cast(current + give_up_samples)); + } + output.resize(static_cast(plan.output_samples)); + for (size_t i = 0; i < output.size(); ++i) { + if (weights[i] <= 0.0f) { + throw std::runtime_error("ZipEnhancer segmented synthesis produced an uncovered sample"); } + output[i] /= weights[i]; } - output.resize(static_cast(original_samples)); return ZipEnhancerWaveformOutput{kSampleRate, std::move(output)}; } diff --git a/tests/unittests/test_enhancement_alignment.cpp b/tests/unittests/test_enhancement_alignment.cpp new file mode 100644 index 000000000..a247770a4 --- /dev/null +++ b/tests/unittests/test_enhancement_alignment.cpp @@ -0,0 +1,442 @@ +#include "engine/framework/audio/flashsr.h" +#include "engine/framework/audio/rnnoise.h" +#include "engine/framework/audio/zipenhancer.h" + +#include "test_assert.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using engine::test::require; +using engine::test::require_close; +using engine::test::require_eq; + +int64_t ceil_div(int64_t value, int64_t divisor) { + return (value + divisor - 1) / divisor; +} + +float to_db(float ratio) { + return 20.0f * std::log10(std::max(ratio, 1.0e-30f)); +} + +std::string with_length(const std::string & label, int64_t samples) { + std::ostringstream oss; + oss << label << " (n=" << samples << ")"; + return oss.str(); +} + +// --------------------------------------------------------------------------- +// F4.18 — RNNoise output is delayed 20 ms and never compensated +// --------------------------------------------------------------------------- + +void test_rnnoise_alignment_plan_arithmetic() { + constexpr int64_t frame = 480; + require_eq(engine::audio::kRnnoiseOutputDelaySamples, static_cast(960), "RNNoise delay constant"); + + const std::vector lengths = {1, 479, 480, 481, 960, 961, 4800, 5760, 48000, 48001, 123457}; + for (const int64_t samples : lengths) { + const auto fixed = engine::audio::rnnoise_alignment_plan(samples, engine::audio::RnnoiseProcessOptions{}); + require_eq(fixed.crop_offset, static_cast(960), with_length("RNNoise crop offset", samples)); + require_eq(fixed.output_samples, samples, with_length("RNNoise output length", samples)); + require_eq(fixed.frames, ceil_div(samples + 960, frame), with_length("RNNoise frame count", samples)); + require_eq(fixed.padded_samples, fixed.frames * frame, with_length("RNNoise padded length", samples)); + require_eq(fixed.vad_frames, ceil_div(samples, frame), with_length("RNNoise vad frame count", samples)); + require( + fixed.padded_samples >= fixed.crop_offset + fixed.output_samples, + with_length("RNNoise padding must cover the delay-compensated crop", samples)); + + // Legacy mode must reproduce the pre-fix arithmetic exactly: frames = + // ceil(n / 480), no crop, output resized straight back to n. + engine::audio::RnnoiseProcessOptions legacy; + legacy.compensate_output_delay = false; + const auto uncompensated = engine::audio::rnnoise_alignment_plan(samples, legacy); + require_eq(uncompensated.crop_offset, static_cast(0), with_length("RNNoise legacy crop", samples)); + require_eq(uncompensated.frames, ceil_div(samples, frame), with_length("RNNoise legacy frames", samples)); + require_eq( + uncompensated.padded_samples, + uncompensated.frames * frame, + with_length("RNNoise legacy padded length", samples)); + require_eq(uncompensated.output_samples, samples, with_length("RNNoise legacy output length", samples)); + } +} + +// Model the analysis/synthesis pair as the pure 960-sample delay it is: +// raw_output[m] == padded_input[m - 960]. The crop the plan describes must undo +// it exactly, for an impulse anywhere in the input including the very last +// sample (which never reached the output before the fix). +void test_rnnoise_crop_recovers_input_alignment() { + const std::vector lengths = {480, 1000, 4800, 5760, 20001}; + for (const int64_t samples : lengths) { + for (const int64_t impulse : {static_cast(0), samples / 3, samples - 1}) { + const auto plan = engine::audio::rnnoise_alignment_plan(samples, engine::audio::RnnoiseProcessOptions{}); + std::vector padded_input(static_cast(plan.padded_samples), 0.0f); + padded_input[static_cast(impulse)] = 1.0f; + + std::vector raw_output(static_cast(plan.padded_samples), 0.0f); + for (int64_t m = engine::audio::kRnnoiseOutputDelaySamples; m < plan.padded_samples; ++m) { + raw_output[static_cast(m)] = + padded_input[static_cast(m - engine::audio::kRnnoiseOutputDelaySamples)]; + } + + require( + plan.crop_offset + plan.output_samples <= plan.padded_samples, + with_length("RNNoise crop fits inside the synthesis buffer", samples)); + const std::vector aligned( + raw_output.begin() + static_cast(plan.crop_offset), + raw_output.begin() + static_cast(plan.crop_offset + plan.output_samples)); + require_eq( + static_cast(aligned.size()), + samples, + with_length("RNNoise aligned output length", samples)); + require_close(aligned[static_cast(impulse)], 1.0f, 0.0f, "RNNoise impulse lands at its input index"); + + double stray = 0.0; + for (size_t i = 0; i < aligned.size(); ++i) { + if (i != static_cast(impulse)) { + stray += static_cast(std::fabs(aligned[i])); + } + } + require(stray == 0.0, with_length("RNNoise aligned output has no stray energy", samples)); + } + } + + // Without compensation the same impulse comes back 960 samples late, and an + // impulse in the final 960 samples of the input is lost entirely. + engine::audio::RnnoiseProcessOptions legacy; + legacy.compensate_output_delay = false; + const int64_t samples = 4800; + const auto plan = engine::audio::rnnoise_alignment_plan(samples, legacy); + std::vector padded_input(static_cast(plan.padded_samples), 0.0f); + padded_input[static_cast(samples - 1)] = 1.0f; + std::vector raw_output(static_cast(plan.padded_samples), 0.0f); + for (int64_t m = engine::audio::kRnnoiseOutputDelaySamples; m < plan.padded_samples; ++m) { + raw_output[static_cast(m)] = + padded_input[static_cast(m - engine::audio::kRnnoiseOutputDelaySamples)]; + } + const std::vector uncompensated( + raw_output.begin(), + raw_output.begin() + static_cast(plan.output_samples)); + float peak = 0.0f; + for (const float value : uncompensated) { + peak = std::max(peak, std::fabs(value)); + } + require(peak == 0.0f, "RNNoise legacy path drops the final 20 ms of input"); +} + +// --------------------------------------------------------------------------- +// F4.16 — ZipEnhancer hard-splices its output every 1.5 seconds +// --------------------------------------------------------------------------- + +void test_zipenhancer_fade_envelope_is_complementary() { + constexpr int64_t overlap = 8000; + for (const int64_t fade : {static_cast(0), static_cast(80), static_cast(320), + static_cast(4000), overlap, overlap * 2}) { + float previous = -1.0f; + for (int64_t position = 0; position < overlap; ++position) { + const float rising = engine::audio::zipenhancer_chunk_fade_weight(position, overlap, fade); + const float falling = engine::audio::zipenhancer_chunk_fade_weight(overlap - 1 - position, overlap, fade); + require(rising >= previous, "ZipEnhancer crossfade envelope must be non-decreasing"); + require(rising >= 0.0f && rising <= 1.0f, "ZipEnhancer crossfade envelope must stay in [0,1]"); + require_close(rising + falling, 1.0f, 1.0e-6f, "ZipEnhancer crossfade pair must sum to unity"); + previous = rising; + } + } + + // A fade of zero is exactly the legacy hard splice: full weight on the + // earlier chunk for the first half of the overlap, then a one-sample step. + require_close(engine::audio::zipenhancer_chunk_fade_weight(3999, 8000, 0), 0.0f, 0.0f, "legacy splice below step"); + require_close(engine::audio::zipenhancer_chunk_fade_weight(4000, 8000, 0), 1.0f, 0.0f, "legacy splice above step"); +} + +// A level mismatch of amplitude d across the join produces a single-sample +// discontinuity of d with a hard splice. Spreading it over a linear fade of F +// samples reduces the largest single-sample step to d / (F + 1), i.e. an +// attenuation of 20*log10(F + 1) dB. +void test_zipenhancer_fade_attenuates_the_seam_step() { + constexpr int64_t overlap = 8000; + struct FadeCase { + int64_t fade; + float min_attenuation_db; + }; + // 20*log10(F + 1): 38.2 dB at 80 samples, 50.1 dB at 320, 78.1 dB at 8000. + const FadeCase cases[] = { + {80, 37.0f}, // 5 ms at 16 kHz + {320, 49.0f}, // 20 ms + {overlap, 77.0f}, // the shipped default: the whole 500 ms overlap + }; + + const float hard_splice_step = + engine::audio::zipenhancer_chunk_fade_weight(4000, overlap, 0) - + engine::audio::zipenhancer_chunk_fade_weight(3999, overlap, 0); + require_close(hard_splice_step, 1.0f, 0.0f, "hard splice step is the full seam amplitude"); + + for (const auto & fade_case : cases) { + float max_step = 0.0f; + for (int64_t position = 1; position < overlap; ++position) { + const float step = + engine::audio::zipenhancer_chunk_fade_weight(position, overlap, fade_case.fade) - + engine::audio::zipenhancer_chunk_fade_weight(position - 1, overlap, fade_case.fade); + max_step = std::max(max_step, std::fabs(step)); + } + const float attenuation_db = -to_db(max_step / hard_splice_step); + std::ostringstream oss; + oss << "ZipEnhancer " << fade_case.fade << "-sample fade seam attenuation " << attenuation_db + << " dB is below the required " << fade_case.min_attenuation_db << " dB"; + require(attenuation_db >= fade_case.min_attenuation_db, oss.str()); + } +} + +// --------------------------------------------------------------------------- +// F4.17 — ZipEnhancer leaves up to 250 ms of silence on the end of the file +// --------------------------------------------------------------------------- + +// Coverage the pre-fix code produced: chunk 0 wrote [0, window - give_up) and +// every later chunk wrote [start + give_up, start + window - give_up). The last +// give_up samples of the padded buffer were never written by anyone. +std::vector legacy_hard_splice_coverage(const engine::audio::ZipEnhancerChunkPlan & plan) { + const int64_t give_up = (plan.window_samples - plan.stride_samples) / 2; + std::vector covered(static_cast(plan.padded_samples), 0.0f); + for (int64_t current = 0; current + plan.window_samples <= plan.padded_samples; current += plan.stride_samples) { + const int64_t begin = current == 0 ? 0 : current + give_up; + const int64_t end = current + plan.window_samples - give_up; + for (int64_t i = begin; i < end; ++i) { + covered[static_cast(i)] = 1.0f; + } + } + return covered; +} + +int64_t trailing_zero_run(const std::vector & values, int64_t limit) { + int64_t run = 0; + for (int64_t i = limit - 1; i >= 0 && values[static_cast(i)] == 0.0f; --i) { + ++run; + } + return run; +} + +void test_zipenhancer_segmented_coverage_has_no_tail_hole() { + // 32000-sample window, 24000-sample stride: remainder == 0 whenever + // n = 32000 + k * 24000, which is the case that used to leave a full 4000 + // zero samples (250 ms) on the end. Sweep it and its neighbours. + std::vector lengths; + for (int64_t k = 4; k <= 9; ++k) { + const int64_t aligned = 32000 + k * 24000; + for (const int64_t delta : {static_cast(-20001), static_cast(-3999), + static_cast(-1), static_cast(0), + static_cast(1), static_cast(3999), + static_cast(20001)}) { + lengths.push_back(aligned + delta); + } + } + lengths.push_back(96001); + lengths.push_back(2880000); // 3 minutes + + bool saw_legacy_hole = false; + for (const int64_t samples : lengths) { + const engine::audio::ZipEnhancerOptions options; + const auto plan = engine::audio::zipenhancer_chunk_plan(samples, options); + require(plan.segmented, with_length("ZipEnhancer sweep length must be segmented", samples)); + require_eq(plan.output_samples, samples, with_length("ZipEnhancer output length", samples)); + require( + plan.padded_samples >= samples, + with_length("ZipEnhancer padded length must cover the input", samples)); + require_eq( + (plan.padded_samples - plan.window_samples) % plan.stride_samples, + static_cast(0), + with_length("ZipEnhancer padded length must land on a stride boundary", samples)); + + const auto weights = engine::audio::zipenhancer_chunk_weights(plan, options); + require_eq( + static_cast(weights.size()), + plan.padded_samples, + with_length("ZipEnhancer weight buffer length", samples)); + + // Every returned sample must be covered, and the accumulated weight must + // be exactly unity so the overlap-add neither dips nor bumps the level. + float min_weight = weights.empty() ? 0.0f : weights[0]; + float max_deviation = 0.0f; + for (int64_t i = 0; i < plan.output_samples; ++i) { + const float weight = weights[static_cast(i)]; + min_weight = std::min(min_weight, weight); + max_deviation = std::max(max_deviation, std::fabs(weight - 1.0f)); + } + require(min_weight > 0.0f, with_length("ZipEnhancer left an uncovered output sample", samples)); + require_close(max_deviation, 0.0f, 1.0e-6f, with_length("ZipEnhancer overlap-add weight is not unity", samples)); + require_eq( + trailing_zero_run(weights, plan.output_samples), + static_cast(0), + with_length("ZipEnhancer trailing uncovered run", samples)); + + // The legacy hard-splice geometry is the regression witness: on the + // stride-aligned lengths it leaves 4000 uncovered samples (250 ms). + const auto legacy_covered = legacy_hard_splice_coverage(plan); + const int64_t legacy_hole = trailing_zero_run(legacy_covered, plan.output_samples); + if (legacy_hole > 0) { + saw_legacy_hole = true; + require( + legacy_hole <= 4000, + with_length("legacy hole should never exceed the give-up region", samples)); + } + + // The legacy crossfade setting must still close the hole. + engine::audio::ZipEnhancerOptions hard_splice; + hard_splice.chunk_crossfade_samples = 0; + const auto hard_weights = engine::audio::zipenhancer_chunk_weights(plan, hard_splice); + for (int64_t i = 0; i < plan.output_samples; ++i) { + require_close( + hard_weights[static_cast(i)], + 1.0f, + 1.0e-6f, + with_length("ZipEnhancer hard-splice weight is not unity", samples)); + } + } + require(saw_legacy_hole, "sweep must include at least one length that used to lose its tail"); +} + +void test_zipenhancer_whole_file_lengths() { + const engine::audio::ZipEnhancerOptions options; + engine::audio::ZipEnhancerOptions legacy; + legacy.match_input_length = false; + legacy.chunk_crossfade_samples = 0; + + // Shorter than the 2 s window: zero-padded up to the window, then trimmed + // back to the input length in both modes. + for (const int64_t samples : {static_cast(1), static_cast(1601), static_cast(31999)}) { + const auto plan = engine::audio::zipenhancer_chunk_plan(samples, options); + require(!plan.segmented, with_length("short input must not segment", samples)); + require_eq(plan.padded_samples, static_cast(32000), with_length("short input padding", samples)); + require_eq(plan.output_samples, samples, with_length("short input output length", samples)); + require_eq( + engine::audio::zipenhancer_chunk_plan(samples, legacy).output_samples, + samples, + with_length("short input legacy output length", samples)); + } + + // Between the window and the 6 s segmentation threshold the un-segmented + // path used to return floor(n / 100) * 100 samples, up to 99 short. + for (const int64_t samples : {static_cast(32000), static_cast(47831), + static_cast(60099), static_cast(96000)}) { + const auto plan = engine::audio::zipenhancer_chunk_plan(samples, options); + require(!plan.segmented, with_length("mid-length input must not segment", samples)); + require_eq(plan.output_samples, samples, with_length("mid-length output length", samples)); + const auto legacy_plan = engine::audio::zipenhancer_chunk_plan(samples, legacy); + require_eq( + legacy_plan.output_samples, + (samples / 100) * 100, + with_length("legacy mid-length output length", samples)); + require( + plan.output_samples - legacy_plan.output_samples <= 99, + with_length("legacy shortfall bound", samples)); + } + + // The exact number the checked-in reference fixture carries. + require_eq( + engine::audio::zipenhancer_chunk_plan(47831, legacy).output_samples, + static_cast(47800), + "legacy path reproduces the reference fixture length"); +} + +// --------------------------------------------------------------------------- +// F4.19 — FlashSR peak-normalises the whole file to 0.999 +// --------------------------------------------------------------------------- + +void test_flashsr_gain_preserves_input_level() { + const engine::audio::FlashSrOptions options; + + // A signal whose peak sits well below full scale must come back at its own + // peak, not at 0.999. + struct LevelCase { + float input_peak; + float output_peak; + }; + const LevelCase cases[] = { + {0.01f, 0.83f}, // -40 dBFS whisper + {0.1f, 0.42f}, + {0.12f, 0.73f}, + {0.5f, 0.2f}, + {0.7071f, 0.9f}, + }; + for (const auto & level : cases) { + const float gain = engine::audio::flashsr_output_gain(level.input_peak, level.output_peak, options); + const float restored = level.output_peak * gain; + std::ostringstream oss; + oss << "FlashSR restored peak for input_peak=" << level.input_peak; + require_close(restored, level.input_peak, 1.0e-6f, oss.str()); + require(restored <= options.peak_ceiling, "FlashSR restored peak must stay under the ceiling"); + + // The legacy policy throws all of that away: every file lands on 0.999 + // regardless of what went in. + engine::audio::FlashSrOptions legacy; + legacy.preserve_input_level = false; + const float legacy_gain = engine::audio::flashsr_output_gain(level.input_peak, level.output_peak, legacy); + require_close(level.output_peak * legacy_gain, options.peak_ceiling, 1.0e-6f, "FlashSR legacy peak"); + } + + // Level must track the input: doubling the input peak doubles the output + // peak. The legacy gain is completely blind to it. + const float quiet = engine::audio::flashsr_output_gain(0.05f, 0.6f, options); + const float loud = engine::audio::flashsr_output_gain(0.10f, 0.6f, options); + require_close(loud / quiet, 2.0f, 1.0e-5f, "FlashSR gain must track the input level"); + const float gain_swing_db = to_db(engine::audio::flashsr_output_gain(0.01f, 0.6f, options)) - + to_db(engine::audio::flashsr_output_gain(0.7071f, 0.6f, options)); + require( + std::fabs(gain_swing_db + 37.0f) < 1.0f, + "FlashSR gain must follow a 37 dB input level swing rather than flatten it"); +} + +void test_flashsr_gain_safety_and_silence() { + const engine::audio::FlashSrOptions options; + + // An input already at or above full scale is limited, never boosted past + // the ceiling. + for (const float input_peak : {1.0f, 1.5f, 12.0f}) { + const float gain = engine::audio::flashsr_output_gain(input_peak, 0.9f, options); + require_close(0.9f * gain, options.peak_ceiling, 1.0e-6f, "FlashSR ceiling clamp"); + } + + // Silence in, silence out — the old code threw here. + require_close(engine::audio::flashsr_output_gain(0.4f, 0.0f, options), 0.0f, 0.0f, "FlashSR zero output peak"); + require_close(engine::audio::flashsr_output_gain(0.0f, 0.5f, options), 0.0f, 0.0f, "FlashSR silent input"); + + // A whole synthetic waveform round-trip: a 0.12-peak sine handed to a model + // that returned a 0.73-peak version of it comes back at 0.12. + std::vector model_output(4800, 0.0f); + for (size_t i = 0; i < model_output.size(); ++i) { + model_output[i] = 0.73f * std::sin(0.05f * static_cast(i)); + } + const float gain = engine::audio::flashsr_output_gain(0.12f, 0.73f, options); + float peak = 0.0f; + for (const float value : model_output) { + peak = std::max(peak, std::fabs(value * gain)); + } + require_close(peak, 0.12f, 1.0e-4f, "FlashSR waveform peak preserved"); +} + +} // namespace + +int main() { + try { + test_rnnoise_alignment_plan_arithmetic(); + test_rnnoise_crop_recovers_input_alignment(); + test_zipenhancer_fade_envelope_is_complementary(); + test_zipenhancer_fade_attenuates_the_seam_step(); + test_zipenhancer_segmented_coverage_has_no_tail_hole(); + test_zipenhancer_whole_file_lengths(); + test_flashsr_gain_preserves_input_level(); + test_flashsr_gain_safety_and_silence(); + std::cout << "enhancement_alignment_test passed\n"; + } catch (const std::exception & ex) { + std::cerr << "enhancement_alignment_test failed: " << ex.what() << "\n"; + return 1; + } + return 0; +} diff --git a/tests/unittests/test_flashsr_utility.cpp b/tests/unittests/test_flashsr_utility.cpp index 4557c34e4..ce52ed0ec 100644 --- a/tests/unittests/test_flashsr_utility.cpp +++ b/tests/unittests/test_flashsr_utility.cpp @@ -72,7 +72,13 @@ void run_case(int case_index) { const auto input = fixture->require_f32_tensor("audio_values"); const auto expected = fixture->require_f32_tensor("reconstruction"); require(input.shape.rank == 2 && input.shape.dims[0] == 1, "FlashSR input shape mismatch"); - const auto output = model.super_resolve_mono_16k(input.values); + // The checked-in fixtures were captured from the upstream reference, which + // peak-normalises every file to the ceiling. Ask for that behaviour + // explicitly so this stays a parity test; input-level preservation is + // covered by enhancement_alignment_test. + engine::audio::FlashSrOptions reference_parity; + reference_parity.preserve_input_level = false; + const auto output = model.super_resolve_mono_16k(input.values, reference_parity); require(output.sample_rate == 48000, "FlashSR sample rate mismatch"); require_close(output.samples, expected, 2.0e-4f, 2.0e-5, "case " + std::to_string(case_index)); } diff --git a/tests/unittests/test_rnnoise_utility.cpp b/tests/unittests/test_rnnoise_utility.cpp index 71034a2c5..c115ba0b2 100644 --- a/tests/unittests/test_rnnoise_utility.cpp +++ b/tests/unittests/test_rnnoise_utility.cpp @@ -104,7 +104,13 @@ void run_waveform_case(const engine::audio::RnnoiseModel & model, const std::str require(input.shape.rank == 2 && input.shape.dims[0] == 1, "RNNoise waveform input shape mismatch"); require(expected_output.shape.rank == 2 && expected_output.shape.dims[0] == 1, "RNNoise waveform output shape mismatch"); require(expected_vad.shape.rank == 2 && expected_vad.shape.dims[0] == 1, "RNNoise waveform vad shape mismatch"); - const auto actual = model.process_mono_48k(input.values); + // The checked-in fixtures were captured from the upstream reference, which + // leaves the 20 ms synthesis delay in the output. Ask for that behaviour + // explicitly so this stays a parity test; delay compensation is covered by + // enhancement_alignment_test. + engine::audio::RnnoiseProcessOptions reference_parity; + reference_parity.compensate_output_delay = false; + const auto actual = model.process_mono_48k(input.values, reference_parity); require(actual.sample_rate == 48000, "RNNoise sample rate mismatch"); require_close( actual.samples, diff --git a/tests/unittests/test_zipenhancer_utility.cpp b/tests/unittests/test_zipenhancer_utility.cpp index 52c8b6a1d..879ca004d 100644 --- a/tests/unittests/test_zipenhancer_utility.cpp +++ b/tests/unittests/test_zipenhancer_utility.cpp @@ -2,6 +2,7 @@ #include "engine/framework/audio/zipenhancer.h" #include +#include #include #include #include @@ -55,11 +56,16 @@ void require_close( double mean_allowed, const std::string & label) { require(expected.shape.rank == 2 && expected.shape.dims[0] == 1, label + " expected shape mismatch"); - require(actual.size() == expected.values.size(), label + " size mismatch"); + // The reference fixtures were captured from the upstream implementation, + // which returns floor(n / 100) * 100 samples. denoise_mono_16k now returns + // the full input length; the samples the fixture does cover are unchanged, + // so compare the overlap and let run_case assert the length separately. + require(actual.size() >= expected.values.size(), label + " size mismatch"); + const size_t compared = expected.values.size(); float max_diff = 0.0f; size_t max_index = 0; double mean_diff = 0.0; - for (size_t i = 0; i < actual.size(); ++i) { + for (size_t i = 0; i < compared; ++i) { const float diff = std::fabs(actual[i] - expected.values[i]); mean_diff += static_cast(diff); if (diff > max_diff) { @@ -67,7 +73,7 @@ void require_close( max_index = i; } } - mean_diff /= static_cast(actual.size()); + mean_diff /= static_cast(compared); if (max_diff > max_allowed || mean_diff > mean_allowed) { std::ostringstream oss; oss << label << " mismatch: max_diff=" << max_diff @@ -87,7 +93,19 @@ void run_case(const engine::audio::ZipEnhancerModel & model, int case_index) { require(input.shape.rank == 2 && input.shape.dims[0] == 1, "ZipEnhancer input shape mismatch"); const auto output = model.denoise_mono_16k(input.values); require(output.sample_rate == 16000, "ZipEnhancer sample rate mismatch"); + require( + output.samples.size() == input.values.size(), + "ZipEnhancer output length must equal its input length in case " + std::to_string(case_index)); require_close(output.samples, expected, 3.0e-3f, 3.0e-4, "case " + std::to_string(case_index)); + + // The legacy length policy must still reproduce the fixture length exactly. + engine::audio::ZipEnhancerOptions legacy; + legacy.match_input_length = false; + legacy.chunk_crossfade_samples = 0; + require( + engine::audio::zipenhancer_chunk_plan(static_cast(input.values.size()), legacy).output_samples == + static_cast(expected.values.size()), + "ZipEnhancer legacy length policy no longer matches the fixture in case " + std::to_string(case_index)); } } // namespace