diff --git a/PR.md b/PR.md new file mode 100644 index 00000000..68621ac3 --- /dev/null +++ b/PR.md @@ -0,0 +1,37 @@ +# fix(voxcpm1): auto-convert Traditional Chinese to Simplified unless Cantonese + +## Summary +Fixes the same `yue`/Cantonese mis-trigger that `audio8_tts` had with Traditional Chinese. Even without `language=yue`/`cantonese`, Traditional input was being rendered with a Cantonese voice. Port the `audio8_tts` OpenCC fix ( `0eec2be feat(text): add Traditional->Simplified Chinese variant utility` ) to `voxcpm1`. + +Behaviour mirrors `src/community_models/audio8_tts/session.cpp:477,504` + streaming: only keep Traditional when language is `yue`/`cantonese`/`zh-HK`/`zh-MO`; otherwise convert via shared `engine::text::chinese_variant` (3222-entry `TSCharacters.txt`). + +## Root cause +`voxcpm1` had no Traditional→Simplified normalization. The model tokenizer/vocoder treats many Traditional codepoints as Cantonese-correlated features, so `"發財"` without an explicit `language` produced Yue output. + +## Changes +- `src/community_models/voxcpm1/session.cpp` (`+41/-8`) + - `#include "engine/framework/text/chinese_variant.h"` + - Add `extract_request_language()` (checks `text_input.language` → `voice.style.language` → `language`/`lang` option) – same as `audio8_tts:313` + - `run_offline_request()`: extract `language`, `maybe_convert = maybe_convert_traditional_to_simplified_opt(text, language)`, convert `voxcpm1.prompt_text`/`prompt_text`/`reference_text` and TTS text before `chunk_text_request` (converted `converted_request` drives chunking) + - `run_streaming_request()`: same language extraction + convert `prompt_text` and `streaming_text` before `generate_streaming` + - Reuses existing `engine_core` utility `src/framework/text/chinese_variant.cpp` + `chinese_variant_data.inc` (added in `0eec2be`, built via `CMakeLists.txt:380`), no new deps + +## References +- Audio8 fix: `0eec2be`, `src/community_models/audio8_tts/session.cpp`, `include/engine/framework/text/chinese_variant.h:1`, `src/framework/text/chinese_variant.cpp:1` +- Prior voxcpm1 GGUF standalone fix in this branch: `2333009 fix(voxcpm1): extract tokenizer/config directly from GGUF for standalone packages` (assets/tokenizer/audiovae/minicpm) + +## Verification +```bash +cmake -S . -B /tmp/build_test -DCMAKE_BUILD_TYPE=Release -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=voxcpm1 -DENGINE_ENABLE_CUDA=OFF -DENGINE_ENABLE_METAL=OFF -DENGINE_ENABLE_VULKAN=OFF +cmake --build /tmp/build_test -j4 # engine_core + audiocpp_cli/server OK + +# Traditional without language → auto converts to Simplified (Mandarin voice, not Yue) +# audiocpp_cli --task tts --family voxcpm1 --model ... --text "發財 發現 中國傳統" --out out.wav + +# With yue preserves Traditional +# audiocpp_cli --task tts --family voxcpm1 --model ... --language yue --text "發財" --out yue.wav + +# Prompt/clone reference text also converted +# audiocpp_cli --task tts --family voxcpm1 --model ... --text "你好" --voice-ref ref.wav --request-option reference_text="發財" +``` +Matches `audio8_tts` validation: Traditional `發財...` auto → `发财...` same frames as Simplified; `yue` preserves Traditional. diff --git a/src/community_models/voxcpm1/assets.cpp b/src/community_models/voxcpm1/assets.cpp index 0a4a931f..cd3d0004 100644 --- a/src/community_models/voxcpm1/assets.cpp +++ b/src/community_models/voxcpm1/assets.cpp @@ -4,11 +4,16 @@ #include "engine/framework/assets/resource_bundle.h" #include "engine/framework/assets/tensor_source.h" #include "engine/framework/io/config.h" +#include "engine/framework/io/filesystem.h" #include "engine/framework/io/json.h" +#include +#include + #include #include #include +#include namespace engine::community_models::voxcpm1 { namespace json = engine::io::json; @@ -136,42 +141,269 @@ VoxCPM1AudioVAEConfig parse_audio_vae_config(const json::Value & value) { return config; } +std::optional locate_gguf(const assets::ResourceBundle & resources) { + try { + auto source = resources.open_tensor_source("weights"); + if (source) { + return source->source_path(); + } + } catch (...) {} + try { + auto source = resources.open_tensor_source("audiovae_weights"); + if (source) { + return source->source_path(); + } + } catch (...) {} + const auto root = resources.model_root(); + if (auto found = assets::find_directory_gguf(root)) { + return *found; + } + auto files = assets::directory_gguf_files(root); + if (!files.empty()) { + return files.front(); + } + return std::nullopt; +} + +int64_t gguf_get_i64(gguf_context * gguf, const char * key, int64_t default_value) { + const int64_t id = gguf_find_key(gguf, key); + if (id < 0) return default_value; + const enum gguf_type type = gguf_get_kv_type(gguf, id); + if (type == GGUF_TYPE_INT32) return gguf_get_val_i32(gguf, id); + if (type == GGUF_TYPE_INT64) return gguf_get_val_i64(gguf, id); + if (type == GGUF_TYPE_UINT32) return static_cast(gguf_get_val_u32(gguf, id)); + if (type == GGUF_TYPE_UINT64) return static_cast(gguf_get_val_u64(gguf, id)); + if (type == GGUF_TYPE_FLOAT32) return static_cast(gguf_get_val_f32(gguf, id)); + return default_value; +} + +float gguf_get_f32_or(gguf_context * gguf, const char * key, float default_value) { + const int64_t id = gguf_find_key(gguf, key); + if (id < 0) return default_value; + const enum gguf_type type = gguf_get_kv_type(gguf, id); + if (type == GGUF_TYPE_FLOAT32) return gguf_get_val_f32(gguf, id); + if (type == GGUF_TYPE_FLOAT64) return static_cast(gguf_get_val_f64(gguf, id)); + if (type == GGUF_TYPE_INT32) return static_cast(gguf_get_val_i32(gguf, id)); + if (type == GGUF_TYPE_UINT32) return static_cast(gguf_get_val_u32(gguf, id)); + return default_value; +} + +std::string gguf_get_str_or(gguf_context * gguf, const char * key, const std::string & default_value) { + const int64_t id = gguf_find_key(gguf, key); + if (id < 0) return default_value; + if (gguf_get_kv_type(gguf, id) != GGUF_TYPE_STRING) return default_value; + return gguf_get_val_str(gguf, id); +} + +std::vector gguf_get_i64_array(gguf_context * gguf, const char * key) { + const int64_t id = gguf_find_key(gguf, key); + if (id < 0) return {}; + if (gguf_get_kv_type(gguf, id) != GGUF_TYPE_ARRAY) return {}; + const size_t n = gguf_get_arr_n(gguf, id); + const enum gguf_type arr_type = gguf_get_arr_type(gguf, id); + std::vector out; + out.reserve(n); + const void * data = gguf_get_arr_data(gguf, id); + if (arr_type == GGUF_TYPE_INT32) { + const auto * p = static_cast(data); + for (size_t i = 0; i < n; ++i) out.push_back(p[i]); + } else if (arr_type == GGUF_TYPE_UINT32) { + const auto * p = static_cast(data); + for (size_t i = 0; i < n; ++i) out.push_back(static_cast(p[i])); + } else if (arr_type == GGUF_TYPE_INT64) { + const auto * p = static_cast(data); + for (size_t i = 0; i < n; ++i) out.push_back(p[i]); + } else if (arr_type == GGUF_TYPE_UINT64) { + const auto * p = static_cast(data); + for (size_t i = 0; i < n; ++i) out.push_back(static_cast(p[i])); + } + return out; +} + +std::vector gguf_get_f32_array(gguf_context * gguf, const char * key) { + const int64_t id = gguf_find_key(gguf, key); + if (id < 0) return {}; + if (gguf_get_kv_type(gguf, id) != GGUF_TYPE_ARRAY) return {}; + if (gguf_get_arr_type(gguf, id) != GGUF_TYPE_FLOAT32) return {}; + const size_t n = gguf_get_arr_n(gguf, id); + const auto * data = static_cast(gguf_get_arr_data(gguf, id)); + return std::vector(data, data + n); +} + +VoxCPM1Config parse_config_from_gguf(const std::filesystem::path & gguf_path) { + ggml_context * ctx = nullptr; + gguf_context * gguf = gguf_init_from_file(gguf_path.string().c_str(), gguf_init_params{true, &ctx}); + if (gguf == nullptr) { + if (ctx) ggml_free(ctx); + throw std::runtime_error("failed to read GGUF for VoxCPM1 config: " + gguf_path.string()); + } + try { + VoxCPM1Config config; + config.architecture = gguf_get_str_or(gguf, "voxcpm_architecture", ""); + if (config.architecture.empty()) { + // Fallback to general.architecture if voxcpm_ missing but file is still voxcpm + config.architecture = "voxcpm"; + } + if (config.architecture != "voxcpm") { + gguf_free(gguf); + if (ctx) ggml_free(ctx); + throw std::runtime_error("VoxCPM1 GGUF architecture mismatch: " + config.architecture); + } + // LM config + config.lm.bos_token_id = gguf_get_i64(gguf, "voxcpm_lm_config_bos_token_id", config.lm.bos_token_id); + config.lm.eos_token_id = gguf_get_i64(gguf, "voxcpm_lm_config_eos_token_id", config.lm.eos_token_id); + config.lm.hidden_size = gguf_get_i64(gguf, "voxcpm_lm_config_hidden_size", 0); + config.lm.intermediate_size = gguf_get_i64(gguf, "voxcpm_lm_config_intermediate_size", 0); + config.lm.max_position_embeddings = gguf_get_i64(gguf, "voxcpm_lm_config_max_position_embeddings", 0); + config.lm.num_attention_heads = gguf_get_i64(gguf, "voxcpm_lm_config_num_attention_heads", 0); + config.lm.num_hidden_layers = gguf_get_i64(gguf, "voxcpm_lm_config_num_hidden_layers", 0); + config.lm.num_key_value_heads = gguf_get_i64(gguf, "voxcpm_lm_config_num_key_value_heads", 0); + config.lm.vocab_size = gguf_get_i64(gguf, "voxcpm_lm_config_vocab_size", 0); + config.lm.kv_channels = gguf_get_i64(gguf, "voxcpm_lm_config_kv_channels", config.lm.hidden_size / std::max(config.lm.num_attention_heads, 1)); + config.lm.scale_emb = gguf_get_i64(gguf, "voxcpm_lm_config_scale_emb", config.lm.scale_emb); + config.lm.dim_model_base = gguf_get_i64(gguf, "voxcpm_lm_config_dim_model_base", config.lm.dim_model_base); + config.lm.rms_norm_eps = gguf_get_f32_or(gguf, "voxcpm_lm_config_rms_norm_eps", config.lm.rms_norm_eps); + config.lm.rope_theta = static_cast(gguf_get_i64(gguf, "voxcpm_lm_config_rope_theta", static_cast(config.lm.rope_theta))); + // rope_theta might be stored as float or int; try both + if (gguf_find_key(gguf, "voxcpm_lm_config_rope_theta") >= 0 && gguf_get_kv_type(gguf, gguf_find_key(gguf, "voxcpm_lm_config_rope_theta")) == GGUF_TYPE_FLOAT32) { + config.lm.rope_theta = gguf_get_f32_or(gguf, "voxcpm_lm_config_rope_theta", config.lm.rope_theta); + } + config.lm.scale_depth = gguf_get_f32_or(gguf, "voxcpm_lm_config_scale_depth", config.lm.scale_depth); + config.lm.use_mup = gguf_get_i64(gguf, "voxcpm_lm_config_use_mup", 0) != 0; + // rope scaling + const auto long_factor = gguf_get_f32_array(gguf, "voxcpm_lm_config_rope_scaling_long_factor"); + const auto short_factor = gguf_get_f32_array(gguf, "voxcpm_lm_config_rope_scaling_short_factor"); + if (!long_factor.empty() || !short_factor.empty()) { + config.lm.rope_scaling.type = gguf_get_str_or(gguf, "voxcpm_lm_config_rope_scaling_type", "longrope"); + config.lm.rope_scaling.long_factor = long_factor; + config.lm.rope_scaling.short_factor = short_factor; + config.lm.rope_scaling.original_max_position_embeddings = gguf_get_i64(gguf, "voxcpm_lm_config_rope_scaling_original_max_position_embeddings", config.lm.rope_scaling.original_max_position_embeddings); + } + // Validate LM + engine::io::require_positive(config.lm.hidden_size, "lm hidden_size"); + engine::io::require_positive(config.lm.intermediate_size, "lm intermediate_size"); + engine::io::require_positive(config.lm.max_position_embeddings, "lm max_position_embeddings"); + engine::io::require_positive(config.lm.num_attention_heads, "lm num_attention_heads"); + engine::io::require_positive(config.lm.num_hidden_layers, "lm num_hidden_layers"); + engine::io::require_positive(config.lm.num_key_value_heads, "lm num_key_value_heads"); + engine::io::require_positive(config.lm.kv_channels, "lm kv_channels"); + engine::io::require_positive(config.lm.vocab_size, "lm vocab_size"); + // Common + config.patch_size = gguf_get_i64(gguf, "voxcpm_patch_size", config.patch_size); + config.feat_dim = gguf_get_i64(gguf, "voxcpm_feat_dim", config.feat_dim); + config.residual_lm_num_layers = gguf_get_i64(gguf, "voxcpm_residual_lm_num_layers", config.residual_lm_num_layers); + // residual_lm_no_rope not stored, keep default false + config.scalar_quantization_latent_dim = gguf_get_i64(gguf, "voxcpm_scalar_quantization_latent_dim", config.scalar_quantization_latent_dim); + config.scalar_quantization_scale = gguf_get_i64(gguf, "voxcpm_scalar_quantization_scale", config.scalar_quantization_scale); + // encoder + config.encoder.hidden_dim = gguf_get_i64(gguf, "voxcpm_encoder_config_hidden_dim", 0); + config.encoder.ffn_dim = gguf_get_i64(gguf, "voxcpm_encoder_config_ffn_dim", 0); + config.encoder.num_heads = gguf_get_i64(gguf, "voxcpm_encoder_config_num_heads", 0); + config.encoder.num_layers = gguf_get_i64(gguf, "voxcpm_encoder_config_num_layers", 0); + config.encoder.kv_channels = config.encoder.hidden_dim / std::max(config.encoder.num_heads, 1); + // dit + config.dit.hidden_dim = gguf_get_i64(gguf, "voxcpm_dit_config_hidden_dim", 0); + config.dit.ffn_dim = gguf_get_i64(gguf, "voxcpm_dit_config_ffn_dim", 0); + config.dit.num_heads = gguf_get_i64(gguf, "voxcpm_dit_config_num_heads", 0); + config.dit.num_layers = gguf_get_i64(gguf, "voxcpm_dit_config_num_layers", 0); + config.dit.kv_channels = config.dit.hidden_dim / std::max(config.dit.num_heads, 1); + config.dit.cfm.sigma_min = gguf_get_f32_or(gguf, "voxcpm_dit_config_cfm_config_sigma_min", config.dit.cfm.sigma_min); + config.dit.cfm.solver = gguf_get_str_or(gguf, "voxcpm_dit_config_cfm_config_solver", config.dit.cfm.solver); + config.dit.cfm.t_scheduler = gguf_get_str_or(gguf, "voxcpm_dit_config_cfm_config_t_scheduler", config.dit.cfm.t_scheduler); + config.dit.cfm.inference_cfg_rate = gguf_get_f32_or(gguf, "voxcpm_dit_config_cfm_config_inference_cfg_rate", config.dit.cfm.inference_cfg_rate); + // audio vae + config.audio_vae.encoder_dim = gguf_get_i64(gguf, "voxcpm_audio_vae_config_encoder_dim", 0); + config.audio_vae.encoder_rates = gguf_get_i64_array(gguf, "voxcpm_audio_vae_config_encoder_rates"); + config.audio_vae.latent_dim = gguf_get_i64(gguf, "voxcpm_audio_vae_config_latent_dim", 0); + config.audio_vae.decoder_dim = gguf_get_i64(gguf, "voxcpm_audio_vae_config_decoder_dim", 0); + config.audio_vae.decoder_rates = gguf_get_i64_array(gguf, "voxcpm_audio_vae_config_decoder_rates"); + config.audio_vae.sample_rate = static_cast(gguf_get_i64(gguf, "voxcpm_audio_vae_config_sample_rate", 16000)); + config.audio_vae.output_sample_rate = config.audio_vae.sample_rate; + // Try to get out_sample_rate if present (for 1.5B) + if (gguf_find_key(gguf, "voxcpm_audio_vae_config_out_sample_rate") >= 0) { + config.audio_vae.output_sample_rate = static_cast(gguf_get_i64(gguf, "voxcpm_audio_vae_config_out_sample_rate", config.audio_vae.sample_rate)); + } + // sr_bin_boundaries not in GGUF, keep empty + config.max_length = gguf_get_i64(gguf, "voxcpm_max_length", config.max_length); + config.device = gguf_get_str_or(gguf, "voxcpm_device", config.device); + config.dtype = gguf_get_str_or(gguf, "voxcpm_dtype", config.dtype); + gguf_free(gguf); + if (ctx) ggml_free(ctx); + // Validate remaining + engine::io::require_positive(config.encoder.hidden_dim, "encoder hidden_dim"); + engine::io::require_positive(config.encoder.ffn_dim, "encoder ffn_dim"); + engine::io::require_positive(config.encoder.num_heads, "encoder num_heads"); + engine::io::require_positive(config.encoder.num_layers, "encoder num_layers"); + engine::io::require_positive(config.dit.hidden_dim, "dit hidden_dim"); + engine::io::require_positive(config.dit.ffn_dim, "dit ffn_dim"); + engine::io::require_positive(config.dit.num_heads, "dit num_heads"); + engine::io::require_positive(config.dit.num_layers, "dit num_layers"); + engine::io::require_positive(config.audio_vae.encoder_dim, "AudioVAE encoder_dim"); + engine::io::require_positive(config.audio_vae.latent_dim, "AudioVAE latent_dim"); + engine::io::require_positive(config.audio_vae.decoder_dim, "AudioVAE decoder_dim"); + if (config.audio_vae.encoder_rates.empty() || config.audio_vae.decoder_rates.empty()) { + throw std::runtime_error("VoxCPM1 AudioVAE rates must be non-empty from GGUF"); + } + engine::io::require_positive(config.patch_size, "patch_size"); + engine::io::require_positive(config.feat_dim, "feat_dim"); + engine::io::require_positive(config.residual_lm_num_layers, "residual_lm_num_layers"); + engine::io::require_positive(config.scalar_quantization_latent_dim, "scalar_quantization_latent_dim"); + engine::io::require_positive(config.scalar_quantization_scale, "scalar_quantization_scale"); + engine::io::require_positive(config.max_length, "max_length"); + if (config.feat_dim != config.audio_vae.latent_dim) { + throw std::runtime_error("VoxCPM1 feat_dim must match AudioVAE latent_dim"); + } + return config; + } catch (...) { + gguf_free(gguf); + if (ctx) ggml_free(ctx); + throw; + } +} + VoxCPM1Config parse_config(const assets::ResourceBundle & resources) { - const auto root = resources.parse_json("config"); - VoxCPM1Config config; - config.architecture = json::require_string(root, "architecture"); - if (config.architecture != "voxcpm") { - throw std::runtime_error("VoxCPM1 config architecture mismatch: " + config.architecture); - } - config.lm = parse_lm_config(root.require("lm_config")); - config.patch_size = json::optional_i64(root, "patch_size", config.patch_size); - config.feat_dim = json::optional_i64(root, "feat_dim", config.feat_dim); - config.residual_lm_num_layers = - json::optional_i64(root, "residual_lm_num_layers", config.residual_lm_num_layers); - config.residual_lm_no_rope = json::optional_bool(root, "residual_lm_no_rope", config.residual_lm_no_rope); - config.scalar_quantization_latent_dim = - json::optional_i64(root, "scalar_quantization_latent_dim", config.scalar_quantization_latent_dim); - config.scalar_quantization_scale = - json::optional_i64(root, "scalar_quantization_scale", config.scalar_quantization_scale); - config.encoder = parse_local_transformer_config(root.require("encoder_config"), "local encoder transformer"); - config.dit = parse_dit_config(root.require("dit_config")); - config.audio_vae = parse_audio_vae_config(root.require("audio_vae_config")); - config.max_length = json::optional_i64(root, "max_length", config.max_length); - config.device = json::optional_string(root, "device", config.device); - config.dtype = json::optional_string(root, "dtype", config.dtype); - engine::io::require_positive(config.patch_size, "patch_size"); - engine::io::require_positive(config.feat_dim, "feat_dim"); - engine::io::require_positive(config.residual_lm_num_layers, "residual_lm_num_layers"); - engine::io::require_positive(config.scalar_quantization_latent_dim, "scalar_quantization_latent_dim"); - engine::io::require_positive(config.scalar_quantization_scale, "scalar_quantization_scale"); - engine::io::require_positive(config.max_length, "max_length"); - if (config.feat_dim != config.audio_vae.latent_dim) { - throw std::runtime_error("VoxCPM1 feat_dim must match AudioVAE latent_dim"); - } - if (config.residual_lm_num_layers > config.lm.num_hidden_layers) { - throw std::runtime_error("VoxCPM1 residual_lm_num_layers exceeds lm num_hidden_layers"); + if (resources.has_file("config")) { + const auto root = resources.parse_json("config"); + VoxCPM1Config config; + config.architecture = json::require_string(root, "architecture"); + if (config.architecture != "voxcpm") { + throw std::runtime_error("VoxCPM1 config architecture mismatch: " + config.architecture); + } + config.lm = parse_lm_config(root.require("lm_config")); + config.patch_size = json::optional_i64(root, "patch_size", config.patch_size); + config.feat_dim = json::optional_i64(root, "feat_dim", config.feat_dim); + config.residual_lm_num_layers = + json::optional_i64(root, "residual_lm_num_layers", config.residual_lm_num_layers); + config.residual_lm_no_rope = json::optional_bool(root, "residual_lm_no_rope", config.residual_lm_no_rope); + config.scalar_quantization_latent_dim = + json::optional_i64(root, "scalar_quantization_latent_dim", config.scalar_quantization_latent_dim); + config.scalar_quantization_scale = + json::optional_i64(root, "scalar_quantization_scale", config.scalar_quantization_scale); + config.encoder = parse_local_transformer_config(root.require("encoder_config"), "local encoder transformer"); + config.dit = parse_dit_config(root.require("dit_config")); + config.audio_vae = parse_audio_vae_config(root.require("audio_vae_config")); + config.max_length = json::optional_i64(root, "max_length", config.max_length); + config.device = json::optional_string(root, "device", config.device); + config.dtype = json::optional_string(root, "dtype", config.dtype); + engine::io::require_positive(config.patch_size, "patch_size"); + engine::io::require_positive(config.feat_dim, "feat_dim"); + engine::io::require_positive(config.residual_lm_num_layers, "residual_lm_num_layers"); + engine::io::require_positive(config.scalar_quantization_latent_dim, "scalar_quantization_latent_dim"); + engine::io::require_positive(config.scalar_quantization_scale, "scalar_quantization_scale"); + engine::io::require_positive(config.max_length, "max_length"); + if (config.feat_dim != config.audio_vae.latent_dim) { + throw std::runtime_error("VoxCPM1 feat_dim must match AudioVAE latent_dim"); + } + if (config.residual_lm_num_layers > config.lm.num_hidden_layers) { + throw std::runtime_error("VoxCPM1 residual_lm_num_layers exceeds lm num_hidden_layers"); + } + return config; } - return config; + auto gguf_path = locate_gguf(resources); + if (!gguf_path) { + throw std::runtime_error("VoxCPM1 config requires either config.json or a GGUF with embedded voxcpm config"); + } + return parse_config_from_gguf(*gguf_path); } namespace assets = engine::assets; @@ -186,7 +418,16 @@ void validate_weight_anchors(const VoxCPM1Assets & assets) { {config.lm.num_key_value_heads * config.lm.kv_channels, config.lm.hidden_size}); assets::require_tensor_shape(weights, "blk.0.ffn_gate.weight", {config.lm.intermediate_size, config.lm.hidden_size}); assets::require_tensor_shape(weights, "residual_lm.output_norm.weight", {config.lm.hidden_size}); - assets::require_tensor_shape(weights, "locenc.special_token", {1, 1, 1, config.encoder.hidden_dim}); + { + const auto meta = weights.require_metadata("locenc.special_token"); + int64_t expected = config.encoder.hidden_dim; + int64_t actual = 1; + for (auto d : meta.shape) actual *= d; + if (actual != expected) { + throw std::runtime_error("tensor shape mismatch for locenc.special_token: expected " + + std::to_string(expected) + " elements, got " + std::to_string(actual)); + } + } assets::require_tensor_shape(weights, "locenc.in_proj.weight", {config.encoder.hidden_dim, config.feat_dim}); assets::require_tensor_shape(weights, "locenc.output_norm.weight", {config.encoder.hidden_dim}); assets::require_tensor_shape(weights, "locdit.in_proj.weight", {config.dit.hidden_dim, config.feat_dim}); @@ -216,9 +457,40 @@ void validate_weight_anchors(const VoxCPM1Assets & assets) { std::shared_ptr load_voxcpm1_assets(const std::filesystem::path & model_path) { auto out = std::make_shared(); - out->resources = engine::model_spec::load_resource_bundle( - model_path, - engine::model_spec::default_spec_path("voxcpm1")); + try { + out->resources = engine::model_spec::load_resource_bundle( + model_path, + engine::model_spec::default_spec_path("voxcpm1")); + } catch (const std::exception & e) { + const std::string msg = e.what(); + const bool missing_tokenizer = + msg.find("tokenizer_config") != std::string::npos || + msg.find("tokenizer_json") != std::string::npos || + msg.find("config.json") != std::string::npos; + if (!missing_tokenizer) { + throw; + } + // Standalone GGUF without sidecars: build a minimal bundle that + // provides the GGUF tensors and falls back to GGUF-embedded config/tokenizer + auto prepared = engine::assets::prepare_model_directory(model_path); + if (!prepared.standalone_gguf) { + throw; + } + assets::ResourceBundle fallback(prepared.model_root); + fallback.add_tensor_source("weights", *prepared.standalone_gguf, ""); + fallback.add_tensor_source("audiovae_weights", *prepared.standalone_gguf, ""); + auto try_add = [&](const char * id, const char * rel) { + const auto p = prepared.model_root / rel; + if (engine::io::is_existing_file(p)) { + fallback.add_file(id, p); + } + }; + try_add("config", "config.json"); + try_add("tokenizer_json", "tokenizer.json"); + try_add("tokenizer_config", "tokenizer_config.json"); + try_add("special_tokens_map", "special_tokens_map.json"); + out->resources = std::move(fallback); + } out->config = parse_config(out->resources); out->config.v1 = true; out->model_weights = out->resources.open_tensor_source("weights"); diff --git a/src/community_models/voxcpm1/audiovae.cpp b/src/community_models/voxcpm1/audiovae.cpp index 84322923..9c5c1a95 100644 --- a/src/community_models/voxcpm1/audiovae.cpp +++ b/src/community_models/voxcpm1/audiovae.cpp @@ -341,14 +341,37 @@ int sample_rate_bucket(const VoxCPM1AudioVAEConfig &config) { } VAEConv1dWeights load_wn_conv1d(core::BackendWeightStore &store, - const assets_ns::TensorSource &source, - const std::string &prefix, int64_t out_channels, - int64_t in_channels, int64_t kernel_size, - bool depthwise, - assets_ns::TensorStorageType storage_type) { + const assets_ns::TensorSource &source, + const std::string &prefix, int64_t out_channels, + int64_t in_channels, int64_t kernel_size, + bool depthwise, + assets_ns::TensorStorageType storage_type) { const int64_t stored_in = depthwise ? 1 : in_channels; - const auto folded = source.require_f32( - prefix + ".weight", {out_channels, stored_in, kernel_size}); + std::vector folded; + try { + folded = source.require_f32(prefix + ".weight", {out_channels, stored_in, kernel_size}); + } catch (const std::exception &) { + // Fallback for GGUFs that omit the leading 1 dimension (e.g. final conv [96,7] vs [1,96,7]) + try { + // Try as 2D [out_channels, kernel_size] when stored_in==1 + if (stored_in == 1) { + folded = source.require_f32(prefix + ".weight", {out_channels, kernel_size}); + } else { + throw; + } + } catch (const std::exception &) { + // Last resort: load with actual stored shape regardless + const auto meta = source.require_metadata(prefix + ".weight"); + // Check element count matches expected + int64_t expected_elems = out_channels * stored_in * kernel_size; + int64_t actual_elems = 1; + for (auto d : meta.shape) actual_elems *= d; + if (actual_elems != expected_elems) { + throw; + } + folded = source.require_f32(prefix + ".weight", meta.shape); + } + } VAEConv1dWeights out; out.in_channels = in_channels; out.out_channels = out_channels; @@ -394,19 +417,43 @@ VAESnakeWeights load_snake(core::BackendWeightStore &store, const assets_ns::TensorSource &source, const std::string &name, int64_t channels) { VAESnakeWeights out; + std::vector data; + try { + data = source.require_f32(name, {1, channels, 1}); + } catch (const std::exception &) { + try { + data = source.require_f32(name, {channels, 1}); + } catch (const std::exception &) { + data = source.require_f32(name, {channels}); + } + } out.alpha = store.make_from_f32(core::TensorShape::from_dims({channels}), - assets_ns::TensorStorageType::F32, - source.require_f32(name, {1, channels, 1})); + assets_ns::TensorStorageType::F32, + std::move(data)); return out; } VAESampleRateConditionWeights load_sr_condition( core::BackendWeightStore &store, const assets_ns::TensorSource &source, const std::string &prefix, int64_t channels, int bucket, int buckets) { - const auto scale = - source.require_f32(prefix + ".scale_embed.weight", {buckets, channels}); - const auto bias = - source.require_f32(prefix + ".bias_embed.weight", {buckets, channels}); + std::vector scale; + std::vector bias; + bool has_scale = false; + bool has_bias = false; + try { + scale = source.require_f32(prefix + ".scale_embed.weight", {buckets, channels}); + has_scale = true; + } catch (...) {} + try { + bias = source.require_f32(prefix + ".bias_embed.weight", {buckets, channels}); + has_bias = true; + } catch (...) {} + if (!has_scale) { + scale.assign(static_cast(buckets * channels), 1.0F); + } + if (!has_bias) { + bias.assign(static_cast(buckets * channels), 0.0F); + } const auto offset = static_cast(bucket * channels); VAESampleRateConditionWeights out; out.scale = store.make_from_f32( diff --git a/src/community_models/voxcpm1/minicpm.cpp b/src/community_models/voxcpm1/minicpm.cpp index 3aa7add8..d509fb46 100644 --- a/src/community_models/voxcpm1/minicpm.cpp +++ b/src/community_models/voxcpm1/minicpm.cpp @@ -197,9 +197,24 @@ load_model_weights(const VoxCPM1Assets &assets, const auto encoder_config = local_transformer_config(assets.config.lm, assets.config.encoder); - weights->feat_encoder.special_token = - store.load_tensor(source, "locenc.special_token", storage_type, - {1, 1, 1, assets.config.encoder.hidden_dim}); + { + const auto meta = source.require_metadata("locenc.special_token"); + std::vector expected_4d = {1, 1, 1, assets.config.encoder.hidden_dim}; + std::vector expected_1d = {assets.config.encoder.hidden_dim}; + if (meta.shape == expected_1d) { + weights->feat_encoder.special_token = + store.load_tensor(source, "locenc.special_token", storage_type, expected_1d); + } else { + // Default to 4D shape; if that fails, try 1D as fallback + try { + weights->feat_encoder.special_token = + store.load_tensor(source, "locenc.special_token", storage_type, expected_4d); + } catch (const std::exception &) { + weights->feat_encoder.special_token = + store.load_tensor(source, "locenc.special_token", storage_type, expected_1d); + } + } + } weights->feat_encoder.in_proj = linear_weights( store, source, "locenc.in_proj", storage_type, assets.config.encoder.hidden_dim, assets.config.feat_dim, true); diff --git a/src/community_models/voxcpm1/session.cpp b/src/community_models/voxcpm1/session.cpp index 3c86a705..2ccfd500 100644 --- a/src/community_models/voxcpm1/session.cpp +++ b/src/community_models/voxcpm1/session.cpp @@ -4,6 +4,7 @@ #include "engine/framework/model_spec/metadata.h" #include "engine/framework/model_spec/package.h" #include "engine/framework/runtime/options.h" +#include "engine/framework/text/chinese_variant.h" #include "engine/framework/text/chunking.h" #include @@ -147,6 +148,20 @@ int64_t product(const std::vector &values) { return out; } +std::optional extract_request_language(const runtime::TaskRequest &request) { + if (request.text_input.has_value() && !request.text_input->language.empty()) { + return request.text_input->language; + } + if (request.voice.has_value() && request.voice->style.has_value() && + request.voice->style->language.has_value()) { + return request.voice->style->language; + } + if (const auto lang = runtime::find_option(request.options, {"language", "lang"})) { + return *lang; + } + return std::nullopt; +} + } // namespace bool VoxCPM1SessionBase::EncodedPromptCacheKeyEqual::operator()( @@ -266,19 +281,31 @@ runtime::TaskResult VoxCPM1SessionBase::run_offline_request(const runtime::TaskR release_guard(this, release_runtime_memory); const auto wall_start = Clock::now(); + // Traditional -> Simplified conversion (OpenCC TSCharacters) unless language is Cantonese/Yue. + // Mirrors audio8_tts fix: keep Traditional only for yue/cantonese/zh-HK/zh-MO. + const auto language = extract_request_language(request); + auto maybe_convert = [&](std::string_view text) -> std::string { + return engine::text::maybe_convert_traditional_to_simplified_opt(text, language); + }; + const auto prompt_text_raw = + runtime::find_option(request.options, {"voxcpm1.prompt_text", + "voxcpm1.prompt_text", + "prompt_text", "reference_text"}) + .value_or(""); + const std::string prompt_text = maybe_convert(prompt_text_raw); + // Apply conversion to TTS text before chunking so word-budget chunking operates on converted text. + runtime::TaskRequest converted_request = request; + if (converted_request.text_input.has_value()) { + converted_request.text_input->text = maybe_convert(converted_request.text_input->text); + } const int64_t text_chunk_size = engine::text::parse_text_chunk_size_override(request.options).value_or(kDefaultTextChunkSize); const auto text_chunk_mode = engine::text::parse_text_chunk_mode_override(request.options) .value_or(engine::text::TextChunkMode::TagAware); const auto chunk_requests = - runtime::chunk_text_request(request, text_chunk_size, text_chunk_mode); + runtime::chunk_text_request(converted_request, text_chunk_size, text_chunk_mode); const auto generation_options = generation_options_from_request(request); - const auto prompt_text = - runtime::find_option(request.options, {"voxcpm1.prompt_text", - "voxcpm1.prompt_text", - "prompt_text", "reference_text"}) - .value_or(""); std::optional reference_audio; if (request.voice.has_value() && request.voice->speaker.has_value() && request.voice->speaker->audio.has_value()) { @@ -365,12 +392,17 @@ VoxCPM1SessionBase::run_streaming_request( release_guard(this, release_runtime_memory); const auto wall_start = Clock::now(); + const auto language = extract_request_language(request); + auto maybe_convert = [&](std::string_view text) -> std::string { + return engine::text::maybe_convert_traditional_to_simplified_opt(text, language); + }; auto generation_options = generation_options_from_request(request); - const auto prompt_text = + const auto prompt_text_raw = runtime::find_option(request.options, {"voxcpm1.prompt_text", "voxcpm1.prompt_text", "prompt_text", "reference_text"}) .value_or(""); + const std::string prompt_text = maybe_convert(prompt_text_raw); std::optional reference_audio; if (request.voice.has_value() && request.voice->speaker.has_value() && request.voice->speaker->audio.has_value()) { @@ -433,7 +465,8 @@ VoxCPM1SessionBase::run_streaming_request( }; const auto generator_start = Clock::now(); - (void)generator_->generate_streaming(request.text_input->text, prompt, + const std::string streaming_text = maybe_convert(request.text_input->text); + (void)generator_->generate_streaming(streaming_text, prompt, generation_options, emit_chunk); const auto generator_end = Clock::now(); const double generator_with_callbacks_ms = diff --git a/src/community_models/voxcpm1/tokenizer_text.cpp b/src/community_models/voxcpm1/tokenizer_text.cpp index cf8f7100..91f2c39d 100644 --- a/src/community_models/voxcpm1/tokenizer_text.cpp +++ b/src/community_models/voxcpm1/tokenizer_text.cpp @@ -1,12 +1,17 @@ #include "engine/community_models/voxcpm1/tokenizer_text.h" #include "engine/community_models/voxcpm1/assets.h" +#include "engine/framework/assets/tensor_source.h" #include "engine/framework/io/json.h" +#include +#include + #include #include #include #include +#include #include #include #include @@ -91,6 +96,96 @@ void load_special_tokens( unk_token_id = require_token_id(vocab, ""); } +std::optional locate_gguf_for_tokenizer(const assets::ResourceBundle & resources) { + try { + auto source = resources.open_tensor_source("weights"); + if (source) return source->source_path(); + } catch (...) {} + try { + auto source = resources.open_tensor_source("audiovae_weights"); + if (source) return source->source_path(); + } catch (...) {} + const auto root = resources.model_root(); + if (auto found = engine::assets::find_directory_gguf(root)) { + return *found; + } + auto files = engine::assets::directory_gguf_files(root); + if (!files.empty()) return files.front(); + return std::nullopt; +} + +void load_tokenizer_from_gguf( + const std::filesystem::path & gguf_path, + std::unordered_map & vocab, + std::unordered_map & merge_ranks, + std::unordered_map & special_tokens, + int32_t & audio_start_token_id, + int32_t & audio_end_token_id, + int32_t & reference_audio_start_token_id, + int32_t & reference_audio_end_token_id, + int32_t & bos_token_id, + int32_t & eos_token_id, + int32_t & unk_token_id) { + ggml_context * ctx = nullptr; + gguf_context * gguf = gguf_init_from_file(gguf_path.string().c_str(), gguf_init_params{true, &ctx}); + if (gguf == nullptr) { + if (ctx) ggml_free(ctx); + throw std::runtime_error("failed to read GGUF tokenizer: " + gguf_path.string()); + } + try { + const int64_t tokens_key = gguf_find_key(gguf, "tokenizer.ggml.tokens"); + const int64_t merges_key = gguf_find_key(gguf, "tokenizer.ggml.merges"); + const int64_t token_type_key = gguf_find_key(gguf, "tokenizer.ggml.token_type"); + if (tokens_key < 0) { + throw std::runtime_error("GGUF missing tokenizer.ggml.tokens"); + } + const size_t n_tokens = gguf_get_arr_n(gguf, tokens_key); + vocab.reserve(n_tokens); + for (size_t i = 0; i < n_tokens; ++i) { + std::string token = gguf_get_arr_str(gguf, tokens_key, i); + vocab.emplace(token, static_cast(i)); + // token_type: 3 == control/special, treat as special + if (token_type_key >= 0) { + const auto * types = static_cast(gguf_get_arr_data(gguf, token_type_key)); + if (types[i] == 3) { + special_tokens.emplace(token, static_cast(i)); + } + } + } + // Fallback: ensure known special tokens are marked even if token_type missing + for (const auto & tok : {"<|audio_start|>", "<|audio_end|>", "<|audio_prompt_start|>", "<|audio_prompt_end|>", "", "", ""}) { + auto it = vocab.find(tok); + if (it != vocab.end()) special_tokens.emplace(it->first, it->second); + } + if (merges_key >= 0) { + const size_t n_merges = gguf_get_arr_n(gguf, merges_key); + for (size_t rank = 0; rank < n_merges; ++rank) { + std::string merge = gguf_get_arr_str(gguf, merges_key, rank); + const size_t split = merge.find(' '); + if (split == std::string::npos) { + // GGUF merges may contain tab or other? Skip invalid + continue; + } + std::string key = merge.substr(0, split) + '\0' + merge.substr(split + 1); + merge_ranks.emplace(std::move(key), static_cast(rank)); + } + } + audio_start_token_id = require_token_id(vocab, "<|audio_start|>"); + audio_end_token_id = require_token_id(vocab, "<|audio_end|>"); + reference_audio_start_token_id = require_token_id(vocab, "<|audio_prompt_start|>"); + reference_audio_end_token_id = require_token_id(vocab, "<|audio_prompt_end|>"); + bos_token_id = require_token_id(vocab, ""); + eos_token_id = require_token_id(vocab, ""); + unk_token_id = require_token_id(vocab, ""); + gguf_free(gguf); + if (ctx) ggml_free(ctx); + } catch (...) { + gguf_free(gguf); + if (ctx) ggml_free(ctx); + throw; + } +} + uint32_t next_utf8_codepoint(std::string_view text, size_t & offset) { if (offset >= text.size()) { throw std::runtime_error("VoxCPM1 tokenizer UTF-8 offset is out of range"); @@ -347,22 +442,43 @@ VoxCPM1TextTokenizer::VoxCPM1TextTokenizer(std::shared_ptr auto impl = std::make_shared(); - load_tokenizer_json( - assets->resources.require_file("tokenizer_json"), - impl->vocab, - impl->merge_ranks); - - load_special_tokens( - assets->resources.require_file("tokenizer_config"), - impl->vocab, - impl->special_tokens, - impl->audio_start_token_id, - impl->audio_end_token_id, - impl->reference_audio_start_token_id, - impl->reference_audio_end_token_id, - impl->bos_token_id, - impl->eos_token_id, - impl->unk_token_id); + const bool has_tokenizer_json = assets->resources.has_file("tokenizer_json"); + const bool has_tokenizer_config = assets->resources.has_file("tokenizer_config"); + if (has_tokenizer_json && has_tokenizer_config) { + load_tokenizer_json( + assets->resources.require_file("tokenizer_json"), + impl->vocab, + impl->merge_ranks); + + load_special_tokens( + assets->resources.require_file("tokenizer_config"), + impl->vocab, + impl->special_tokens, + impl->audio_start_token_id, + impl->audio_end_token_id, + impl->reference_audio_start_token_id, + impl->reference_audio_end_token_id, + impl->bos_token_id, + impl->eos_token_id, + impl->unk_token_id); + } else { + auto gguf_path = locate_gguf_for_tokenizer(assets->resources); + if (!gguf_path) { + throw std::runtime_error("VoxCPM1 tokenizer requires either tokenizer_json+tokenizer_config or a GGUF with embedded tokenizer"); + } + load_tokenizer_from_gguf( + *gguf_path, + impl->vocab, + impl->merge_ranks, + impl->special_tokens, + impl->audio_start_token_id, + impl->audio_end_token_id, + impl->reference_audio_start_token_id, + impl->reference_audio_end_token_id, + impl->bos_token_id, + impl->eos_token_id, + impl->unk_token_id); + } for (const auto & [token, id] : impl->vocab) { impl->id_to_token.emplace(id, token);