Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1935,6 +1935,17 @@ if (ENGINE_BUILD_WARMBENCH)
add_engine_warmbench(voxcpm2_warm_bench tests/voxcpm2/voxcpm2_warm_bench.cpp)
endif()

option(AUDIOCPP_BUILD_ENHANCE_TOOL "Build the audiocpp_enhance audio-utility CLI" OFF)
if (AUDIOCPP_BUILD_ENHANCE_TOOL)
add_executable(audiocpp_enhance
tools/audiocpp_enhance/main.cpp
)
target_link_libraries(audiocpp_enhance PRIVATE engine_runtime ggml)
if (ENGINE_ENABLE_OPENMP)
target_link_libraries(audiocpp_enhance PRIVATE OpenMP::OpenMP_CXX)
endif()
endif()

if (ENGINE_BUILD_TESTS)
enable_testing()

Expand Down Expand Up @@ -2032,6 +2043,20 @@ if (ENGINE_BUILD_TESTS)
COMMAND audio_chunking_test
)

add_engine_unittest(audio_resample_quality_test tests/unittests/test_audio_resample_quality.cpp)

add_test(
NAME audio_resample_quality_test
COMMAND audio_resample_quality_test
)

add_engine_unittest(wav_writer_formats_test tests/unittests/test_wav_writer_formats.cpp)

add_test(
NAME wav_writer_formats_test
COMMAND wav_writer_formats_test
)

add_engine_unittest(chinese_normalization_test tests/unittests/test_chinese_normalization.cpp)

add_test(
Expand Down
34 changes: 34 additions & 0 deletions docs/audio_tools.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,40 @@ Common CLI shape:
audiocpp_cli --task <task> --family <family> --model <model-dir> --backend cuda ...
```

## audiocpp_enhance

`audiocpp_enhance` is an optional CLI over the `engine::audio` helpers that are
otherwise library-only: the denoisers, FlashSR super-resolution, and resampling.
It is useful for preparing reference audio and for post-processing generated
output without writing a program against the framework.

It is not built by default:

```bash
cmake -S . -B build -DAUDIOCPP_BUILD_ENHANCE_TOOL=ON
cmake --build build --target audiocpp_enhance
```

```bash
# denoise
build/bin/audiocpp_enhance --backend metal --denoise zipenhancer \
--in noisy.wav --out clean.wav

# super-resolve 16 kHz speech to 48 kHz
build/bin/audiocpp_enhance --backend metal --flashsr --in narrow.wav --out wide.wav

# resample, preserving channel count
build/bin/audiocpp_enhance --resample 48000 --in mix_44k.wav --out mix_48k.wav
```

`--backend` accepts `cpu` (default) or `metal`. Denoise models are `zipenhancer`,
`deepfilternet2` and `rnnoise`; each needs its package installed.

Resampling handles any channel count, resampling each channel independently and
re-interleaving. It goes through `resample_mono_soxr_or_sinc`, which uses libsoxr
when it is loadable and the in-tree windowed sinc otherwise, so it never silently
degrades to linear interpolation. `--in` and `--out` must be different paths.

## AudioSR

AudioSR performs audio super-resolution from an input waveform. See
Expand Down
22 changes: 22 additions & 0 deletions include/engine/framework/audio/conversion.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,4 +50,26 @@ std::vector<float> read_wav_f32_as_mono_linear_resampled(
const std::filesystem::path & path,
int target_sample_rate_hz);

// Anti-aliased equivalents of the three helpers above. The `_linear_` versions
// call a two-tap interpolator with no decimation filter, so a 48 -> 16 kHz
// conversion folds a 12 kHz tone back to 4 kHz at -9.5 dBc; these route through
// soxr, or the in-tree windowed sinc at playback width, and measure -126.7 dBc
// for the same job. Use them wherever the audio is destined for a listener or
// for a model that is expected to see a clean band. The `_linear_` versions are
// kept unchanged for the call sites that assert parity against a reference
// implementation.
std::vector<float> convert_wav_to_mono_quality_resampled(
const WavData & wav,
int target_sample_rate_hz);

std::vector<float> convert_interleaved_audio_to_mono_quality_resampled(
const std::vector<float> & interleaved_samples,
int sample_rate_hz,
int channel_count,
int target_sample_rate_hz);

std::vector<float> read_wav_f32_as_mono_quality_resampled(
const std::filesystem::path & path,
int target_sample_rate_hz);

} // namespace engine::audio
24 changes: 24 additions & 0 deletions include/engine/framework/audio/output.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#pragma once

#include "engine/framework/audio/wav_writer.h"

#include <filesystem>
#include <string>
#include <vector>
Expand All @@ -19,10 +21,32 @@ class IAudioSink {
virtual void write(const std::filesystem::path & path, const AudioBuffer & audio) const = 0;
};

// 16-bit PCM, hard clip, no dither. Kept as its own type because it is the
// default sink for every existing route.
class WavPcm16Sink final : public IAudioSink {
public:
std::string family() const override;
void write(const std::filesystem::path & path, const AudioBuffer & audio) const override;
};

// Bit-depth-selectable sink. Default-constructed it is byte-for-byte the same
// output as WavPcm16Sink; construct it with WavWriteOptions to emit 24-bit PCM
// or float32, or to enable dither or the soft peak limiter.
class WavSink final : public IAudioSink {
public:
WavSink() = default;
explicit WavSink(const WavWriteOptions & options)
: options_(options) {}

const WavWriteOptions & options() const noexcept {
return options_;
}

std::string family() const override;
void write(const std::filesystem::path & path, const AudioBuffer & audio) const override;

private:
WavWriteOptions options_{};
};

} // namespace engine::audio
26 changes: 26 additions & 0 deletions include/engine/framework/audio/resampling.h
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,18 @@ std::vector<float> resample_mono_soxr_or_linear(
int target_sample_rate_hz,
const SoxrResampleOptions & options);

// Same contract as resample_mono_soxr_or_linear, but the fallback is the
// in-tree windowed-sinc resampler at playback width rather than a two-tap
// linear interpolator, so output quality does not depend on whether an optional
// system library happens to be installed. Use this on any path whose output a
// listener will hear. The requested output_length_policy is applied to the
// fallback result too, which the linear fallback does not do.
std::vector<float> resample_mono_soxr_or_sinc(
const std::vector<float> & mono_samples,
int source_sample_rate_hz,
int target_sample_rate_hz,
const SoxrResampleOptions & options);

std::vector<float> resample_mono_linear(
const std::vector<float> & mono_samples,
int source_sample_rate_hz,
Expand All @@ -57,6 +69,14 @@ enum class TorchaudioSincHannAccumulation {
};

struct TorchaudioSincHannResampleOptions {
// 6 is torchaudio.transforms.Resample's own default and is deliberately
// kept: the ~36 call sites that take it are feature-extraction paths whose
// job is bit-parity with a Python reference, and widening the kernel there
// would change every one of their model inputs. It is the wrong width for
// audible output — a 44.1 -> 48 -> 44.1 kHz music round trip measures
// 74.1 dB at width 6 against 130.5 dB at width 64, and the worst alias
// image from a 19.5 kHz tone is -13.8 dBc against -80.8 dBc. Playback paths
// should ask for torchaudio_sinc_hann_playback_options() instead.
int64_t lowpass_filter_width = 6;
double rolloff = 0.99;
TorchaudioSincHannKernelMode kernel_mode = TorchaudioSincHannKernelMode::Float64ComputationStoredAsFloat32;
Expand All @@ -65,6 +85,12 @@ struct TorchaudioSincHannResampleOptions {

TorchaudioSincHannResampleOptions torchaudio_sinc_hann_float32_options();

// Width 64. The measured knee for music: 44.1 -> 48 -> 44.1 kHz round trip is
// 74.1 dB at width 6, 101.1 dB at 16, 130.5 dB at 64, 129.6 dB at 256, so 64
// buys 56 dB over the default and 256 buys nothing further. Cost scales roughly
// linearly with width.
TorchaudioSincHannResampleOptions torchaudio_sinc_hann_playback_options();

std::vector<float> resample_mono_torchaudio_sinc_hann(
const std::vector<float> & mono_samples,
int source_sample_rate_hz,
Expand Down
81 changes: 81 additions & 0 deletions include/engine/framework/audio/wav_writer.h
Original file line number Diff line number Diff line change
@@ -1,10 +1,91 @@
#pragma once

#include <cstdint>
#include <filesystem>
#include <vector>

namespace engine::audio {

// Output sample formats the WAV writer can emit. The reader in wav_reader.h
// decodes PCM 8/16/24/32, float 32/64, A-law and mu-law; these are the three
// worth writing. 16-bit is the historical behaviour and stays the default so no
// existing caller changes.
enum class WavSampleFormat {
Pcm16,
Pcm24,
Float32,
};

// Dither is a deliberate choice, not a universal improvement. It trades raw SNR
// for decorrelated quantisation error: measured on a -60 dBFS 997 Hz sine,
// undithered rounding leaves harmonics at -54.6 dBc while 2 LSB peak-to-peak
// TPDF pushes them to -66.1 dBc, at the cost of 4.6 dB of SNR. It must be
// applied exactly once, at the final quantisation to an integer format, so it
// stays opt-in: a chain that writes an intermediate 16-bit file and reads it
// back would otherwise dither twice.
enum class WavDitherMode {
None,
TriangularPdf,
};

// What to do with samples outside +/-1.0. HardClip is the historical behaviour:
// a per-sample clamp, which on a tone peaking 1 dB over full scale puts 30 % of
// samples on the rail and measures -26.7 dB THD+N. LookaheadLimit instead
// applies a smoothed broadband gain envelope that dips before the overshoot
// arrives, which on the same signal reaches the ceiling with 0 % of samples
// railed and -113 dB THD+N, at the cost of about 1 dB of level.
//
// A memoryless soft-clip waveshaper was tried first and rejected: shaping the
// top of the waveform measured -26.7 dB THD+N, indistinguishable from the hard
// clamp it replaced. Overshoot is a gain problem, not a curve problem.
enum class WavPeakPolicy {
HardClip,
LookaheadLimit,
};

struct LookaheadLimiterOptions {
// Just under full scale, so the int quantiser below never rounds up onto
// the rail.
float ceiling = 0.995F;
// Lookahead and release half-window. 5 ms is long enough to ride over a
// 200 Hz cycle without audible pumping and short enough that a single
// transient does not duck a whole bar.
double window_seconds = 0.005;
};

struct WavWriteOptions {
WavSampleFormat format = WavSampleFormat::Pcm16;
WavDitherMode dither = WavDitherMode::None;
WavPeakPolicy peak_policy = WavPeakPolicy::HardClip;
LookaheadLimiterOptions limiter{};
// Dither is generated from a deterministic per-call sequence so a written
// file is reproducible; change the seed to decorrelate repeated renders.
uint64_t dither_seed = 0x9E3779B97F4A7C15ULL;
};

// Smoothed lookahead peak limiter, for callers that need to manage peaks before
// the output stage (a summed mix, for example). Channels are gain-linked from a
// per-frame peak so the stereo image does not shift. Returns the largest gain
// reduction applied, in dB; returns 0 and leaves the buffer bit-identical when
// nothing exceeded the ceiling.
float apply_lookahead_limiter_in_place(
std::vector<float> & samples,
int channel_count,
int sample_rate,
const LookaheadLimiterOptions & options = {});

int wav_sample_format_bit_depth(WavSampleFormat format);
const char * wav_sample_format_name(WavSampleFormat format);

void write_wav(
const std::filesystem::path & path,
int sample_rate,
int channel_count,
const std::vector<float> & audio,
const WavWriteOptions & options = {});

// 16-bit PCM, hard clip, no dither. Unchanged behaviour for the ~70 call sites
// that use it; equivalent to write_wav with default options.
void write_pcm16_wav(
const std::filesystem::path & path,
int sample_rate,
Expand Down
33 changes: 33 additions & 0 deletions src/framework/audio/conversion.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -149,4 +149,37 @@ std::vector<float> read_wav_f32_as_mono_linear_resampled(
return convert_wav_to_mono_linear_resampled(read_wav_f32(path), target_sample_rate_hz);
}

std::vector<float> convert_wav_to_mono_quality_resampled(
const WavData & wav,
int target_sample_rate_hz) {
if (wav.sample_rate <= 0 || target_sample_rate_hz <= 0) {
throw std::runtime_error("audio sample rates must be positive");
}
auto mono = mixdown_interleaved_to_mono_average(wav.samples, wav.channels);
if (wav.sample_rate != target_sample_rate_hz) {
SoxrResampleOptions options;
options.profile = SoxrResampleProfile::QualityOnly;
options.warning_context = "audio input conversion";
options.fallback_description = "windowed-sinc resampling";
mono = resample_mono_soxr_or_sinc(mono, wav.sample_rate, target_sample_rate_hz, options);
}
return mono;
}

std::vector<float> convert_interleaved_audio_to_mono_quality_resampled(
const std::vector<float> & interleaved_samples,
int sample_rate_hz,
int channel_count,
int target_sample_rate_hz) {
return convert_wav_to_mono_quality_resampled(
WavData{sample_rate_hz, channel_count, interleaved_samples},
target_sample_rate_hz);
}

std::vector<float> read_wav_f32_as_mono_quality_resampled(
const std::filesystem::path & path,
int target_sample_rate_hz) {
return convert_wav_to_mono_quality_resampled(read_wav_f32(path), target_sample_rate_hz);
}

} // namespace engine::audio
10 changes: 8 additions & 2 deletions src/framework/audio/mixing.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,14 @@ std::vector<float> resample_interleaved_to_rate(
options.profile = SoxrResampleProfile::QualityOnly;
options.output_length_policy = SoxrOutputLengthPolicy::ExactExpected;
options.warning_context = "audio mix";
options.fallback_description = "linear resampling";
auto resampled = resample_mono_soxr_or_linear(mono, source_rate, target_rate, options);
options.fallback_description = "windowed-sinc resampling";
// This is the audible mix output. The old fallback was a two-tap linear
// interpolator, which round-trips music at 41 dB SNR against 130 dB for
// the windowed sinc, and which ignored output_length_policy entirely —
// it sized with llround where soxr sizes with ceil, so a channel could
// come back one sample short and trip the length check below purely on
// whether libsoxr happened to be installed.
auto resampled = resample_mono_soxr_or_sinc(mono, source_rate, target_rate, options);
if (output_frames < 0) {
output_frames = static_cast<int64_t>(resampled.size());
} else if (output_frames != static_cast<int64_t>(resampled.size())) {
Expand Down
10 changes: 9 additions & 1 deletion src/framework/audio/output.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

#include "engine/framework/audio/wav_writer.h"

#include <stdexcept>
#include <string>

namespace engine::audio {

Expand All @@ -14,4 +14,12 @@ void WavPcm16Sink::write(const std::filesystem::path & path, const AudioBuffer &
write_pcm16_wav(path, audio.sample_rate, audio.channel_count, audio.samples);
}

std::string WavSink::family() const {
return std::string("wav_") + wav_sample_format_name(options_.format);
}

void WavSink::write(const std::filesystem::path & path, const AudioBuffer & audio) const {
write_wav(path, audio.sample_rate, audio.channel_count, audio.samples, options_);
}

} // namespace engine::audio
Loading
Loading