diff --git a/Cargo.lock b/Cargo.lock index c35d42e34..c0d4ce7ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1163,7 +1163,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llama-cpp-bindings" -version = "0.12.0" +version = "0.13.0" dependencies = [ "encoding_rs", "enumflags2", @@ -1182,27 +1182,28 @@ dependencies = [ [[package]] name = "llama-cpp-bindings-build" -version = "0.12.0" +version = "0.13.0" dependencies = [ "bindgen", "cc", "cmake", "find_cuda_helper", "glob", + "serial_test", "thiserror", "walkdir", ] [[package]] name = "llama-cpp-bindings-sys" -version = "0.12.0" +version = "0.13.0" dependencies = [ "llama-cpp-bindings-build", ] [[package]] name = "llama-cpp-bindings-tests" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "encoding_rs", @@ -1214,7 +1215,7 @@ dependencies = [ [[package]] name = "llama-cpp-bindings-types" -version = "0.12.0" +version = "0.13.0" dependencies = [ "serde", "serde_json", @@ -1223,11 +1224,11 @@ dependencies = [ [[package]] name = "llama-cpp-error-recorder" -version = "0.12.0" +version = "0.13.0" [[package]] name = "llama-cpp-gbnf" -version = "0.12.0" +version = "0.13.0" dependencies = [ "llama-cpp-bindings-sys", "thiserror", @@ -1235,11 +1236,11 @@ dependencies = [ [[package]] name = "llama-cpp-log-decoder" -version = "0.12.0" +version = "0.13.0" [[package]] name = "llama-cpp-test-harness" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "hf-hub", @@ -1252,7 +1253,7 @@ dependencies = [ [[package]] name = "llama-cpp-test-harness-macros" -version = "0.12.0" +version = "0.13.0" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 45335b3a6..306ab80cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ members = [ [workspace.package] edition = "2024" -version = "0.12.0" +version = "0.13.0" license = "Apache-2.0" repository = "https://github.com/intentee/llama-cpp-bindings" @@ -31,15 +31,15 @@ glob = "=0.3.3" hf-hub = "=0.5.0" inventory = "=0.3.24" libtest-mimic = "=0.8.2" -llama-cpp-bindings = { path = "llama-cpp-bindings", version = "=0.12.0" } -llama-cpp-bindings-build = { path = "llama-cpp-bindings-build", version = "=0.12.0" } -llama-cpp-bindings-sys = { path = "llama-cpp-bindings-sys", version = "=0.12.0" } -llama-cpp-bindings-types = { path = "llama-cpp-bindings-types", version = "=0.12.0" } -llama-cpp-error-recorder = { path = "llama-cpp-error-recorder", version = "=0.12.0" } -llama-cpp-gbnf = { path = "llama-cpp-gbnf", version = "=0.12.0" } -llama-cpp-log-decoder = { path = "llama-cpp-log-decoder", version = "=0.12.0" } -llama-cpp-test-harness = { path = "llama-cpp-test-harness", version = "=0.12.0" } -llama-cpp-test-harness-macros = { path = "llama-cpp-test-harness-macros", version = "=0.12.0" } +llama-cpp-bindings = { path = "llama-cpp-bindings", version = "=0.13.0" } +llama-cpp-bindings-build = { path = "llama-cpp-bindings-build", version = "=0.13.0" } +llama-cpp-bindings-sys = { path = "llama-cpp-bindings-sys", version = "=0.13.0" } +llama-cpp-bindings-types = { path = "llama-cpp-bindings-types", version = "=0.13.0" } +llama-cpp-error-recorder = { path = "llama-cpp-error-recorder", version = "=0.13.0" } +llama-cpp-gbnf = { path = "llama-cpp-gbnf", version = "=0.13.0" } +llama-cpp-log-decoder = { path = "llama-cpp-log-decoder", version = "=0.13.0" } +llama-cpp-test-harness = { path = "llama-cpp-test-harness", version = "=0.13.0" } +llama-cpp-test-harness-macros = { path = "llama-cpp-test-harness-macros", version = "=0.13.0" } llguidance = "=1.7.0" log = "=0.4.29" nom = "=8.0.0" diff --git a/llama-cpp-bindings-build/Cargo.toml b/llama-cpp-bindings-build/Cargo.toml index 801cb01d9..3129dc4ff 100644 --- a/llama-cpp-bindings-build/Cargo.toml +++ b/llama-cpp-bindings-build/Cargo.toml @@ -15,6 +15,9 @@ glob = { workspace = true } thiserror = { workspace = true } walkdir = { workspace = true } +[dev-dependencies] +serial_test = { workspace = true } + [features] cuda = [] cuda-no-vmm = ["cuda"] diff --git a/llama-cpp-bindings-build/src/android_ndk.rs b/llama-cpp-bindings-build/src/android_ndk.rs index 0d0123c19..3621ae617 100644 --- a/llama-cpp-bindings-build/src/android_ndk.rs +++ b/llama-cpp-bindings-build/src/android_ndk.rs @@ -163,14 +163,17 @@ fn detect_api_level() -> String { } fn detect_host_tag() -> Result<&'static str, AndroidNdkDetectionError> { - if cfg!(target_os = "macos") { - Ok("darwin-x86_64") - } else if cfg!(target_os = "linux") { - Ok("linux-x86_64") - } else if cfg!(target_os = "windows") { - Ok("windows-x86_64") - } else { - Err(AndroidNdkDetectionError::UnsupportedHostPlatform) + host_tag_for_os(std::env::consts::OS) +} + +/// Resolved from an OS name rather than a `cfg!` branch, so every host's tag +/// stays reachable from any machine. +fn host_tag_for_os(os: &str) -> Result<&'static str, AndroidNdkDetectionError> { + match os { + "macos" => Ok("darwin-x86_64"), + "linux" => Ok("linux-x86_64"), + "windows" => Ok("windows-x86_64"), + _ => Err(AndroidNdkDetectionError::UnsupportedHostPlatform), } } @@ -233,3 +236,353 @@ fn find_clang_builtin_includes(toolchain_path: &str) -> Option { None } } + +#[cfg(test)] +mod tests { + use std::path::Path; + + use serial_test::serial; + + use crate::scratch_dir::ScratchDir; + + use super::AndroidNdk; + use super::AndroidNdkDetectionError; + use super::detect_api_level; + use super::detect_host_tag; + use super::detect_ndk_from_sdk; + use super::detect_ndk_path; + use super::find_clang_builtin_includes; + use super::host_tag_for_os; + use super::target_triple_to_abi; + use super::target_triple_to_ndk_prefix; + use super::validate_ndk_installation; + + const NDK_VARS: &[&str] = &[ + "ANDROID_NDK", + "ANDROID_NDK_ROOT", + "NDK_ROOT", + "CARGO_NDK_ANDROID_NDK", + "ANDROID_HOME", + "ANDROID_SDK_ROOT", + "ANDROID_API_LEVEL", + "ANDROID_PLATFORM", + "CARGO_NDK_ANDROID_PLATFORM", + ]; + + fn clear_ndk_environment() { + for name in NDK_VARS { + unsafe { std::env::remove_var(name) }; + } + } + + fn host_tag() -> &'static str { + detect_host_tag().expect("the test host must be supported") + } + + /// A synthetic NDK layout: enough structure for detection to succeed. + fn ndk_tree(scratch: &ScratchDir, with_clang_includes: bool) -> String { + let ndk = scratch.path().join("ndk/27.0.1"); + std::fs::create_dir_all(ndk.join("build/cmake")).expect("cmake dir must be creatable"); + std::fs::write(ndk.join("build/cmake/android.toolchain.cmake"), b"x") + .expect("toolchain file must be writable"); + let toolchain = ndk.join(format!("toolchains/llvm/prebuilt/{}", host_tag())); + std::fs::create_dir_all(toolchain.join("sysroot")).expect("sysroot must be creatable"); + + if with_clang_includes { + std::fs::create_dir_all(toolchain.join("lib/clang/18/include")) + .expect("clang include dir must be creatable"); + } + + ndk.to_string_lossy().into_owned() + } + + #[test] + fn each_supported_architecture_maps_to_an_abi_and_prefix() { + for (triple, abi, prefix) in [ + ( + "aarch64-linux-android", + "arm64-v8a", + "aarch64-linux-android", + ), + ( + "armv7-linux-androideabi", + "armeabi-v7a", + "arm-linux-androideabi", + ), + ("x86_64-linux-android", "x86_64", "x86_64-linux-android"), + ("i686-linux-android", "x86", "i686-linux-android"), + ] { + assert_eq!(target_triple_to_abi(triple).expect(triple), abi); + assert_eq!(target_triple_to_ndk_prefix(triple).expect(triple), prefix); + } + } + + #[test] + fn an_unsupported_architecture_is_rejected_by_both_mappings() { + assert!(matches!( + target_triple_to_abi("mips-linux-android"), + Err(AndroidNdkDetectionError::UnsupportedAndroidTarget { .. }) + )); + assert!(matches!( + target_triple_to_ndk_prefix("mips-linux-android"), + Err(AndroidNdkDetectionError::UnsupportedAndroidTarget { .. }) + )); + } + + #[test] + fn every_supported_host_has_a_tag_and_others_are_rejected() { + assert_eq!(host_tag_for_os("macos").expect("macos"), "darwin-x86_64"); + assert_eq!(host_tag_for_os("linux").expect("linux"), "linux-x86_64"); + assert_eq!( + host_tag_for_os("windows").expect("windows"), + "windows-x86_64" + ); + assert!(matches!( + host_tag_for_os("plan9"), + Err(AndroidNdkDetectionError::UnsupportedHostPlatform) + )); + } + + #[test] + #[serial] + fn the_api_level_falls_back_through_each_variable() { + clear_ndk_environment(); + assert_eq!(detect_api_level(), "28"); + + unsafe { std::env::set_var("CARGO_NDK_ANDROID_PLATFORM", "android-30") }; + assert_eq!(detect_api_level(), "30"); + + unsafe { std::env::set_var("ANDROID_PLATFORM", "android-31") }; + assert_eq!(detect_api_level(), "31"); + + unsafe { std::env::set_var("ANDROID_API_LEVEL", "32") }; + assert_eq!(detect_api_level(), "32"); + + clear_ndk_environment(); + } + + #[test] + #[serial] + fn each_ndk_variable_is_consulted_in_order() { + clear_ndk_environment(); + + for name in [ + "CARGO_NDK_ANDROID_NDK", + "NDK_ROOT", + "ANDROID_NDK_ROOT", + "ANDROID_NDK", + ] { + unsafe { std::env::set_var(name, format!("/from/{name}")) }; + + assert_eq!( + detect_ndk_path("aarch64-linux-android").expect(name), + format!("/from/{name}") + ); + } + + clear_ndk_environment(); + } + + #[test] + #[serial] + fn an_unset_ndk_reports_the_target_triple() { + clear_ndk_environment(); + let scratch = ScratchDir::new("ndk-unset"); + unsafe { std::env::set_var("ANDROID_HOME", scratch.path()) }; + + let error = + detect_ndk_path("aarch64-linux-android").expect_err("no NDK anywhere must fail"); + + clear_ndk_environment(); + + assert!(matches!( + error, + AndroidNdkDetectionError::NdkRootNotConfigured { ref target_triple, .. } + if target_triple == "aarch64-linux-android" + )); + } + + #[test] + #[serial] + fn the_newest_sdk_installed_ndk_is_selected() { + clear_ndk_environment(); + let scratch = ScratchDir::new("ndk-sdk"); + let ndk_dir = scratch.path().join("ndk"); + std::fs::create_dir_all(ndk_dir.join("25.1.0")).expect("older ndk must be creatable"); + std::fs::create_dir_all(ndk_dir.join("27.0.1")).expect("newer ndk must be creatable"); + std::fs::write(ndk_dir.join("not-a-directory"), b"x").expect("file must be writable"); + unsafe { std::env::set_var("ANDROID_SDK_ROOT", scratch.path()) }; + + let detected = detect_ndk_from_sdk().expect("an SDK-installed NDK must be found"); + + clear_ndk_environment(); + + assert!(detected.ends_with("ndk/27.0.1"), "got: {detected}"); + } + + #[test] + #[serial] + fn an_sdk_without_any_ndk_directory_is_not_detected() { + clear_ndk_environment(); + let scratch = ScratchDir::new("ndk-sdk-empty"); + std::fs::create_dir_all(scratch.path().join("ndk")).expect("ndk dir must be creatable"); + unsafe { std::env::set_var("ANDROID_HOME", scratch.path()) }; + + let outcome = detect_ndk_from_sdk(); + + clear_ndk_environment(); + + assert!(outcome.is_err(), "an empty ndk directory yields nothing"); + } + + #[test] + fn validation_rejects_a_missing_root_and_a_missing_toolchain_file() { + let scratch = ScratchDir::new("ndk-validate"); + + assert!(matches!( + validate_ndk_installation("/definitely/not/here"), + Err(AndroidNdkDetectionError::NdkRootMissing { .. }) + )); + + let bare = scratch.path().join("bare"); + std::fs::create_dir_all(&bare).expect("bare ndk must be creatable"); + + assert!(matches!( + validate_ndk_installation(&bare.to_string_lossy()), + Err(AndroidNdkDetectionError::NdkToolchainFileMissing { .. }) + )); + } + + #[test] + fn clang_builtin_includes_are_found_only_when_the_version_directory_exists() { + let scratch = ScratchDir::new("ndk-clang"); + let toolchain = scratch.path().join("toolchain"); + + assert_eq!( + find_clang_builtin_includes(&toolchain.to_string_lossy()), + None, + "a missing lib/clang directory yields nothing" + ); + + std::fs::create_dir_all(toolchain.join("lib/clang")).expect("clang dir must be creatable"); + std::fs::create_dir_all(toolchain.join("lib/clang/not-a-version")) + .expect("non-version dir must be creatable"); + + assert_eq!( + find_clang_builtin_includes(&toolchain.to_string_lossy()), + None, + "only digit-prefixed version directories count" + ); + + std::fs::create_dir_all(toolchain.join("lib/clang/18")) + .expect("version dir must be creatable"); + + assert_eq!( + find_clang_builtin_includes(&toolchain.to_string_lossy()), + None, + "a version directory without include/ yields nothing" + ); + + std::fs::create_dir_all(toolchain.join("lib/clang/18/include")) + .expect("include dir must be creatable"); + + let found = find_clang_builtin_includes(&toolchain.to_string_lossy()) + .expect("a complete layout must resolve"); + + assert!(found.ends_with("lib/clang/18/include"), "got: {found}"); + } + + #[test] + #[serial] + fn a_complete_ndk_layout_is_detected_end_to_end() { + clear_ndk_environment(); + let scratch = ScratchDir::new("ndk-complete"); + let ndk_path = ndk_tree(&scratch, true); + unsafe { std::env::set_var("ANDROID_NDK", &ndk_path) }; + + let ndk = AndroidNdk::detect("aarch64-linux-android").expect("layout must be detected"); + + clear_ndk_environment(); + + assert_eq!(ndk.abi, "arm64-v8a"); + assert_eq!(ndk.target_prefix, "aarch64-linux-android"); + assert_eq!(ndk.api_level, "28"); + assert_eq!(ndk.android_platform(), "android-28"); + assert_eq!( + ndk.cmake_toolchain_file(), + format!("{ndk_path}/build/cmake/android.toolchain.cmake") + ); + assert!(Path::new(&ndk.sysroot).is_dir()); + assert!(ndk.clang_builtin_includes.is_some()); + assert!(format!("{ndk:?}").contains("AndroidNdk")); + } + + #[test] + #[serial] + fn a_layout_without_the_host_toolchain_directory_is_rejected() { + clear_ndk_environment(); + let scratch = ScratchDir::new("ndk-no-toolchain"); + let ndk = scratch.path().join("ndk"); + std::fs::create_dir_all(ndk.join("build/cmake")).expect("cmake dir must be creatable"); + std::fs::write(ndk.join("build/cmake/android.toolchain.cmake"), b"x") + .expect("toolchain file must be writable"); + unsafe { std::env::set_var("ANDROID_NDK", ndk.to_string_lossy().as_ref()) }; + + let error = AndroidNdk::detect("aarch64-linux-android") + .expect_err("a missing host toolchain must fail"); + + clear_ndk_environment(); + + assert!(matches!( + error, + AndroidNdkDetectionError::NdkToolchainDirectoryMissing { .. } + )); + } + + #[test] + #[serial] + fn a_complete_layout_without_clang_includes_still_detects() { + clear_ndk_environment(); + let scratch = ScratchDir::new("ndk-no-clang"); + unsafe { std::env::set_var("ANDROID_NDK", ndk_tree(&scratch, false)) }; + + let ndk = AndroidNdk::detect("x86_64-linux-android").expect("layout must be detected"); + + clear_ndk_environment(); + + assert_eq!(ndk.abi, "x86_64"); + assert_eq!(ndk.clang_builtin_includes, None); + } + + #[test] + fn every_error_variant_renders_a_message() { + let messages = [ + AndroidNdkDetectionError::NdkRootMissing { path: "/p".into() }.to_string(), + AndroidNdkDetectionError::NdkToolchainFileMissing { path: "/p".into() }.to_string(), + AndroidNdkDetectionError::NdkToolchainDirectoryMissing { path: "/p".into() } + .to_string(), + AndroidNdkDetectionError::UnsupportedHostPlatform.to_string(), + AndroidNdkDetectionError::UnsupportedAndroidTarget { + target_triple: "mips".to_owned(), + } + .to_string(), + ]; + + for message in messages { + assert!(!message.is_empty()); + } + } + + #[test] + #[serial] + fn an_unset_sdk_root_falls_back_to_the_home_directory() { + clear_ndk_environment(); + + let outcome = detect_ndk_from_sdk(); + + assert!( + outcome.is_err(), + "no NDK is installed under the home directory in this environment" + ); + } +} diff --git a/llama-cpp-bindings-build/src/bindgen_config.rs b/llama-cpp-bindings-build/src/bindgen_config.rs index 549e0bd87..b734c2024 100644 --- a/llama-cpp-bindings-build/src/bindgen_config.rs +++ b/llama-cpp-bindings-build/src/bindgen_config.rs @@ -6,13 +6,14 @@ use crate::debug_log; use crate::target_os::TargetOs; pub fn generate_bindings( + wrapper_dir: &Path, llama_src: &Path, out_dir: &Path, target_os: &TargetOs, target_triple: &str, android_ndk: Option<&AndroidNdk>, ) { - let mut builder = create_base_builder(llama_src); + let mut builder = create_base_builder(wrapper_dir, llama_src); if target_os.is_android() && let Some(ndk) = android_ndk @@ -35,10 +36,10 @@ pub fn generate_bindings( debug_log!("Bindings Created"); } -fn create_base_builder(llama_src: &Path) -> bindgen::Builder { +fn create_base_builder(wrapper_dir: &Path, llama_src: &Path) -> bindgen::Builder { bindgen::Builder::default() - .header("wrapper.h") - .header("wrapper_mtmd.h") + .header(wrapper_dir.join("wrapper.h").to_string_lossy()) + .header(wrapper_dir.join("wrapper_mtmd.h").to_string_lossy()) .clang_arg(format!("-I{}", llama_src.join("include").display())) .clang_arg(format!("-I{}", llama_src.join("ggml/include").display())) .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) @@ -101,6 +102,30 @@ fn configure_android_bindgen( builder } +fn apply_msvc_include_paths( + mut builder: bindgen::Builder, + include_paths: Option<&str>, +) -> bindgen::Builder { + let Some(include_paths) = include_paths else { + return builder; + }; + + for include_path in split_msvc_include_paths(include_paths) { + builder = builder.clang_arg("-isystem").clang_arg(&include_path); + debug_log!("Added MSVC include path: {}", include_path); + } + + builder +} + +fn split_msvc_include_paths(include_paths: &str) -> Vec { + include_paths + .split(';') + .filter(|path| !path.is_empty()) + .map(str::to_owned) + .collect() +} + fn configure_msvc_bindgen(mut builder: bindgen::Builder, target_triple: &str) -> bindgen::Builder { let out_dir_str = env::var("OUT_DIR").unwrap_or_default(); let dummy_c = Path::new(&out_dir_str).join("dummy.c"); @@ -122,16 +147,13 @@ fn configure_msvc_bindgen(mut builder: bindgen::Builder, target_triple: &str) -> .find(|(key, _)| key.eq_ignore_ascii_case("INCLUDE")) .map(|(_, value)| value.clone()); - if let Some(include_paths) = msvc_include_paths { - for include_path in include_paths - .to_string_lossy() - .split(';') - .filter(|path| !path.is_empty()) - { - builder = builder.clang_arg("-isystem").clang_arg(include_path); - debug_log!("Added MSVC include path: {}", include_path); - } - } + builder = apply_msvc_include_paths( + builder, + msvc_include_paths + .as_ref() + .map(|paths| paths.to_string_lossy()) + .as_deref(), + ); builder = builder .clang_arg(format!("--target={target_triple}")) @@ -145,3 +167,307 @@ fn configure_msvc_bindgen(mut builder: bindgen::Builder, target_triple: &str) -> builder } + +#[cfg(test)] +mod tests { + use serial_test::serial; + + use crate::android_ndk::AndroidNdk; + use crate::cc_test_environment::with_cc_environment; + use crate::scratch_dir::ScratchDir; + + use super::apply_msvc_include_paths; + use super::configure_android_bindgen; + use super::configure_msvc_bindgen; + use super::create_base_builder; + use super::split_msvc_include_paths; + + fn android_ndk(clang_builtin_includes: Option) -> AndroidNdk { + AndroidNdk { + ndk_path: "/ndk".to_owned(), + api_level: "28".to_owned(), + abi: "arm64-v8a", + host_tag: "darwin-x86_64", + toolchain_path: "/ndk/toolchains/llvm/prebuilt/darwin-x86_64".to_owned(), + sysroot: "/ndk/toolchains/llvm/prebuilt/darwin-x86_64/sysroot".to_owned(), + target_prefix: "aarch64-linux-android", + clang_builtin_includes, + } + } + + fn rendered(builder: &bindgen::Builder) -> String { + format!("{builder:?}") + } + + #[test] + #[serial] + fn bindings_are_generated_and_respect_the_allowlist() { + let scratch = ScratchDir::new("bindgen-generate"); + let llama_src = scratch.path().join("llama.cpp"); + std::fs::create_dir_all(llama_src.join("include")).expect("include must be creatable"); + std::fs::create_dir_all(llama_src.join("ggml/include")).expect("ggml must be creatable"); + std::fs::write( + scratch.path().join("wrapper.h"), + b"int llama_probe(void);\nint unrelated_probe(void);\n", + ) + .expect("wrapper header must be writable"); + std::fs::write( + scratch.path().join("wrapper_mtmd.h"), + b"int mtmd_probe(void);\n", + ) + .expect("mtmd header must be writable"); + + super::generate_bindings( + scratch.path(), + &llama_src, + scratch.path(), + &crate::target_os::TargetOs::from_target_triple("aarch64-apple-darwin") + .expect("supported triple"), + "aarch64-apple-darwin", + None, + ); + + let generated = std::fs::read_to_string(scratch.path().join("bindings.rs")) + .expect("bindings must be written"); + + assert!(generated.contains("llama_probe"), "allowlisted llama_*"); + assert!(generated.contains("mtmd_probe"), "allowlisted mtmd_*"); + assert!( + !generated.contains("unrelated_probe"), + "symbols outside the allowlist must be excluded" + ); + } + + #[test] + fn the_base_builder_allowlists_the_upstream_prefixes() { + let builder = + create_base_builder(std::path::Path::new("/sys"), std::path::Path::new("/llama")); + let description = rendered(&builder); + + for expected in ["llama_.*", "ggml_.*", "mtmd_.*", "gguf_.*"] { + assert!(description.contains(expected), "missing {expected}"); + } + assert!(description.contains("/llama/include")); + assert!(description.contains("/llama/ggml/include")); + } + + #[test] + #[serial] + fn the_android_configuration_adds_the_sysroot_and_builtin_includes() { + unsafe { std::env::remove_var("CARGO_SUBCOMMAND") }; + let ndk = android_ndk(Some("/ndk/lib/clang/18/include".to_owned())); + + let builder = configure_android_bindgen( + create_base_builder(std::path::Path::new("/sys"), std::path::Path::new("/llama")), + &ndk, + "aarch64-linux-android", + ); + let description = rendered(&builder); + + assert!(description.contains(&format!("--sysroot={}", ndk.sysroot))); + assert!(description.contains("-D__ANDROID_API__=28")); + assert!(description.contains("/ndk/lib/clang/18/include")); + assert!(description.contains("stdbool.h")); + } + + #[test] + #[serial] + fn the_android_configuration_tolerates_absent_builtin_includes() { + unsafe { std::env::remove_var("CARGO_SUBCOMMAND") }; + + let builder = configure_android_bindgen( + create_base_builder(std::path::Path::new("/sys"), std::path::Path::new("/llama")), + &android_ndk(None), + "aarch64-linux-android", + ); + + assert!(rendered(&builder).contains("-D__ANDROID__")); + } + + #[test] + #[serial] + fn the_cargo_ndk_subcommand_sets_the_extra_clang_target() { + unsafe { + std::env::set_var("CARGO_SUBCOMMAND", "ndk"); + std::env::remove_var("BINDGEN_EXTRA_CLANG_ARGS"); + } + + let _ = configure_android_bindgen( + create_base_builder(std::path::Path::new("/sys"), std::path::Path::new("/llama")), + &android_ndk(None), + "aarch64-linux-android", + ); + + let extra = std::env::var("BINDGEN_EXTRA_CLANG_ARGS").unwrap_or_default(); + + unsafe { + std::env::remove_var("CARGO_SUBCOMMAND"); + std::env::remove_var("BINDGEN_EXTRA_CLANG_ARGS"); + } + + assert_eq!(extra, "--target=aarch64-linux-android"); + } + + #[test] + #[serial] + fn the_msvc_configuration_adds_compatibility_flags() { + let scratch = ScratchDir::new("bindgen-msvc"); + + with_cc_environment(scratch.path(), || { + let builder = configure_msvc_bindgen( + create_base_builder(std::path::Path::new("/sys"), std::path::Path::new("/llama")), + "x86_64-pc-windows-msvc", + ); + let description = rendered(&builder); + + assert!(description.contains("-fms-compatibility")); + assert!(description.contains("-fms-extensions")); + assert!(description.contains("--target=x86_64-pc-windows-msvc")); + }); + } + + #[test] + #[serial] + fn an_unwritable_out_dir_leaves_the_msvc_builder_untouched() { + unsafe { std::env::set_var("OUT_DIR", "/definitely/not/a/directory") }; + + let builder = configure_msvc_bindgen( + create_base_builder(std::path::Path::new("/sys"), std::path::Path::new("/llama")), + "x86_64-pc-windows-msvc", + ); + + unsafe { std::env::remove_var("OUT_DIR") }; + + assert!(!rendered(&builder).contains("-fms-compatibility")); + } + + #[test] + #[serial] + fn an_msvc_target_takes_the_msvc_configuration_path() { + let scratch = ScratchDir::new("bindgen-msvc-path"); + let llama_src = scratch.path().join("llama.cpp"); + std::fs::create_dir_all(llama_src.join("ggml/include")).expect("tree must be creatable"); + std::fs::write( + scratch.path().join("wrapper.h"), + b"int llama_probe(void);\n", + ) + .expect("header must be writable"); + std::fs::write( + scratch.path().join("wrapper_mtmd.h"), + b"int mtmd_probe(void);\n", + ) + .expect("header must be writable"); + + with_cc_environment(scratch.path(), || { + unsafe { std::env::set_var("INCLUDE", format!("{};", scratch.path().display())) }; + + super::generate_bindings( + scratch.path(), + &llama_src, + scratch.path(), + &crate::target_os::TargetOs::from_target_triple("x86_64-pc-windows-msvc") + .expect("msvc triple"), + "x86_64-pc-windows-msvc", + None, + ); + + unsafe { std::env::remove_var("INCLUDE") }; + }); + + assert!( + scratch.path().join("bindings.rs").exists(), + "bindings must still be produced for an msvc target" + ); + } + + #[test] + #[serial] + fn an_android_target_takes_the_ndk_configuration_path() { + let scratch = ScratchDir::new("bindgen-android-path"); + let llama_src = scratch.path().join("llama.cpp"); + std::fs::create_dir_all(llama_src.join("ggml/include")).expect("tree must be creatable"); + std::fs::write( + scratch.path().join("wrapper.h"), + b"int llama_probe(void);\n", + ) + .expect("header must be writable"); + std::fs::write( + scratch.path().join("wrapper_mtmd.h"), + b"int mtmd_probe(void);\n", + ) + .expect("header must be writable"); + let sysroot = scratch.path().join("sysroot"); + std::fs::create_dir_all(sysroot.join("usr/include")).expect("sysroot must be creatable"); + + let mut ndk = android_ndk(None); + ndk.sysroot = sysroot.to_string_lossy().into_owned(); + + unsafe { std::env::remove_var("CARGO_SUBCOMMAND") }; + + super::generate_bindings( + scratch.path(), + &llama_src, + scratch.path(), + &crate::target_os::TargetOs::from_target_triple("aarch64-linux-android") + .expect("android triple"), + "aarch64-linux-android", + Some(&ndk), + ); + + assert!( + scratch.path().join("bindings.rs").exists(), + "bindings must still be produced for an android target" + ); + } + + #[test] + #[serial] + fn a_missing_compiler_environment_leaves_the_msvc_builder_untouched() { + let scratch = ScratchDir::new("bindgen-nocompiler"); + + unsafe { + std::env::set_var("OUT_DIR", scratch.path()); + std::env::remove_var("HOST"); + std::env::remove_var("TARGET"); + std::env::remove_var("OPT_LEVEL"); + } + + let builder = configure_msvc_bindgen( + create_base_builder(std::path::Path::new("/sys"), std::path::Path::new("/llama")), + "x86_64-pc-windows-msvc", + ); + + unsafe { std::env::remove_var("OUT_DIR") }; + + assert!( + !rendered(&builder).contains("-fms-compatibility"), + "without a resolvable compiler the builder must be returned unchanged" + ); + } + + #[test] + fn msvc_include_paths_are_split_and_emptied_entries_dropped() { + assert_eq!( + split_msvc_include_paths("C:\\a;C:\\b;;C:\\c"), + vec!["C:\\a".to_owned(), "C:\\b".to_owned(), "C:\\c".to_owned()] + ); + assert!(split_msvc_include_paths("").is_empty()); + assert!(split_msvc_include_paths(";;").is_empty()); + } + + #[test] + fn msvc_include_paths_are_applied_when_the_toolchain_reports_them() { + let base = + || create_base_builder(std::path::Path::new("/sys"), std::path::Path::new("/llama")); + + let applied = apply_msvc_include_paths(base(), Some("/win/ucrt;/win/shared")); + let description = rendered(&applied); + + assert!(description.contains("/win/ucrt")); + assert!(description.contains("/win/shared")); + + let untouched = rendered(&apply_msvc_include_paths(base(), None)); + + assert!(!untouched.contains("/win/ucrt")); + } +} diff --git a/llama-cpp-bindings-build/src/cc_test_environment.rs b/llama-cpp-bindings-build/src/cc_test_environment.rs new file mode 100644 index 000000000..bd289c541 --- /dev/null +++ b/llama-cpp-bindings-build/src/cc_test_environment.rs @@ -0,0 +1,40 @@ +use std::path::Path; + +/// Runs `body` with the build-script environment the toolchain crates require, +/// returning whatever it produced. +pub fn with_cc_environment_value TValue>( + out_dir: &Path, + body: TBody, +) -> TValue { + let mut produced = None; + + with_cc_environment(out_dir, || produced = Some(body())); + + produced.expect("the body always runs") +} + +/// Runs `body` with the build-script environment `cc` requires to resolve a +/// compiler and place its output. +pub fn with_cc_environment(out_dir: &Path, body: TBody) { + let host = format!("{}-apple-darwin", std::env::consts::ARCH); + + unsafe { + std::env::set_var("OUT_DIR", out_dir); + std::env::set_var("HOST", &host); + std::env::set_var("TARGET", &host); + std::env::set_var("OPT_LEVEL", "0"); + std::env::set_var("PROFILE", "debug"); + std::env::set_var("NUM_JOBS", "1"); + } + + body(); + + unsafe { + std::env::remove_var("OUT_DIR"); + std::env::remove_var("HOST"); + std::env::remove_var("TARGET"); + std::env::remove_var("OPT_LEVEL"); + std::env::remove_var("PROFILE"); + std::env::remove_var("NUM_JOBS"); + } +} diff --git a/llama-cpp-bindings-build/src/cmake_config.rs b/llama-cpp-bindings-build/src/cmake_config.rs index 12faa145a..648d0f8dd 100644 --- a/llama-cpp-bindings-build/src/cmake_config.rs +++ b/llama-cpp-bindings-build/src/cmake_config.rs @@ -361,3 +361,348 @@ fn configure_system_ggml(config: &mut Config) { config.define("LLAMA_USE_SYSTEM_GGML", "ON"); } } + +#[cfg(test)] +mod tests { + use cmake::Config; + use serial_test::serial; + + use crate::android_ndk::AndroidNdk; + use crate::scratch_dir::ScratchDir; + use crate::target_os::TargetOs; + + use super::configure_android_arch_flags; + use super::configure_android_cmake; + use super::configure_base_defines; + use super::configure_compiler_launchers; + use super::configure_cpu_features; + use super::configure_dynamic_backends; + use super::configure_gpu_backends; + use super::configure_msvc_release_workaround; + use super::configure_openmp; + use super::configure_platform_specific; + use super::configure_shared_libs; + use super::configure_system_ggml; + use super::configure_vulkan_linking; + use super::map_cpu_feature_to_ggml; + use super::override_archive_commands_for_apple_ar; + use super::pass_cmake_env_vars; + use super::which; + + fn config() -> Config { + Config::new(".") + } + + fn target_os(triple: &str) -> TargetOs { + TargetOs::from_target_triple(triple).expect("supported triple") + } + + fn android_ndk(abi: &'static str) -> AndroidNdk { + AndroidNdk { + ndk_path: "/ndk".to_owned(), + api_level: "28".to_owned(), + abi, + host_tag: "darwin-x86_64", + toolchain_path: "/ndk/toolchain".to_owned(), + sysroot: "/ndk/toolchain/sysroot".to_owned(), + target_prefix: "aarch64-linux-android", + clang_builtin_includes: None, + } + } + + #[test] + fn every_recognised_cpu_feature_maps_to_a_ggml_flag() { + for (feature, flag) in [ + ("avx", "GGML_AVX"), + ("avx2", "GGML_AVX2"), + ("avx512bf16", "GGML_AVX512_BF16"), + ("avx512vbmi", "GGML_AVX512_VBMI"), + ("avx512vnni", "GGML_AVX512_VNNI"), + ("avxvnni", "GGML_AVX_VNNI"), + ("bmi2", "GGML_BMI2"), + ("f16c", "GGML_F16C"), + ("fma", "GGML_FMA"), + ("sse4.2", "GGML_SSE42"), + ] { + assert_eq!(map_cpu_feature_to_ggml(feature), Some(flag)); + } + + assert_eq!(map_cpu_feature_to_ggml("neon"), None); + } + + #[test] + #[serial] + fn which_finds_a_program_on_path_and_reports_absence() { + let scratch = ScratchDir::new("cmake-which"); + let program = scratch.path().join("a-build-tool"); + std::fs::write(&program, b"#!/bin/sh\n").expect("program must be writable"); + let previous = std::env::var_os("PATH").expect("PATH is always set"); + unsafe { std::env::set_var("PATH", scratch.path()) }; + + let found = which("a-build-tool"); + let missing = which("definitely-not-a-program"); + + unsafe { std::env::set_var("PATH", previous) }; + + assert_eq!(found, Some(program)); + assert_eq!(missing, None); + } + + #[test] + #[serial] + fn dynamic_backends_are_configured_only_behind_the_feature() { + let scratch = ScratchDir::new("cmake-backends"); + let mut config = config(); + + let backends_dir = configure_dynamic_backends(&mut config, scratch.path()); + + if cfg!(feature = "dynamic-backends") { + let dir = backends_dir.expect("the feature must yield a backends directory"); + assert!(dir.is_dir(), "the backends directory must be created"); + } else { + assert_eq!(backends_dir, None); + } + } + + #[test] + #[serial] + fn compiler_launchers_are_skipped_when_ccache_is_disabled() { + unsafe { std::env::set_var("LLAMA_DISABLE_CCACHE", "1") }; + + configure_compiler_launchers(&mut config()); + + unsafe { std::env::remove_var("LLAMA_DISABLE_CCACHE") }; + } + + #[test] + #[serial] + fn compiler_launchers_are_configured_when_ccache_is_present() { + let scratch = ScratchDir::new("cmake-ccache"); + std::fs::write(scratch.path().join("ccache"), b"#!/bin/sh\n") + .expect("ccache stand-in must be writable"); + let previous_path = std::env::var_os("PATH").expect("PATH is always set"); + + unsafe { + std::env::remove_var("LLAMA_DISABLE_CCACHE"); + std::env::set_var("PATH", scratch.path()); + } + + configure_compiler_launchers(&mut config()); + + unsafe { std::env::set_var("PATH", previous_path) }; + } + + #[test] + #[serial] + fn cmake_prefixed_environment_variables_are_forwarded() { + unsafe { std::env::set_var("CMAKE_A_TEST_ONLY_VARIABLE", "value") }; + + pass_cmake_env_vars(&mut config()); + + unsafe { std::env::remove_var("CMAKE_A_TEST_ONLY_VARIABLE") }; + } + + #[test] + #[serial] + fn a_native_target_cpu_enables_ggml_native() { + unsafe { + std::env::set_var("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=native"); + } + + configure_cpu_features(&mut config(), "aarch64-apple-darwin"); + + unsafe { std::env::remove_var("CARGO_ENCODED_RUSTFLAGS") }; + } + + #[test] + #[serial] + fn an_explicit_target_cpu_sets_a_baseline_march() { + unsafe { + std::env::set_var("CARGO_ENCODED_RUSTFLAGS", "-Ctarget-cpu=haswell"); + std::env::set_var("CARGO_CFG_TARGET_FEATURE", "avx,avx2,neon"); + } + + configure_cpu_features(&mut config(), "aarch64-unknown-linux-gnu"); + + unsafe { + std::env::remove_var("CARGO_ENCODED_RUSTFLAGS"); + std::env::remove_var("CARGO_CFG_TARGET_FEATURE"); + } + } + + #[test] + #[serial] + fn an_absent_target_cpu_still_configures_features() { + unsafe { + std::env::remove_var("CARGO_ENCODED_RUSTFLAGS"); + std::env::remove_var("CARGO_CFG_TARGET_FEATURE"); + } + + configure_cpu_features(&mut config(), "x86_64-unknown-linux-gnu"); + } + + #[test] + fn shared_library_configuration_covers_both_settings() { + configure_shared_libs(&mut config(), true); + configure_shared_libs(&mut config(), false); + } + + #[test] + fn every_platform_arm_is_configured() { + configure_platform_specific( + &mut config(), + &target_os("aarch64-apple-darwin"), + "aarch64-apple-darwin", + "Release", + None, + ); + configure_platform_specific( + &mut config(), + &target_os("x86_64-pc-windows-msvc"), + "x86_64-pc-windows-msvc", + "Release", + None, + ); + configure_platform_specific( + &mut config(), + &target_os("x86_64-unknown-linux-gnu"), + "x86_64-unknown-linux-gnu", + "Release", + None, + ); + configure_platform_specific( + &mut config(), + &target_os("aarch64-linux-android"), + "aarch64-linux-android", + "Release", + None, + ); + configure_platform_specific( + &mut config(), + &target_os("aarch64-linux-android"), + "aarch64-linux-android", + "Release", + Some(&android_ndk("arm64-v8a")), + ); + } + + #[test] + fn the_msvc_release_workaround_applies_only_to_release_profiles() { + for profile in ["Release", "RelWithDebInfo", "MinSizeRel", "Debug"] { + configure_msvc_release_workaround(&mut config(), profile); + } + } + + #[test] + fn the_android_cmake_configuration_uses_the_detected_ndk() { + configure_android_cmake( + &mut config(), + &android_ndk("arm64-v8a"), + "aarch64-linux-android", + ); + } + + #[test] + fn every_android_abi_sets_architecture_flags() { + for abi in ["arm64-v8a", "armeabi-v7a", "x86_64", "x86", "riscv64"] { + configure_android_arch_flags(&mut config(), abi); + } + } + + #[test] + fn apple_archive_commands_are_overridden() { + override_archive_commands_for_apple_ar(&mut config()); + } + + #[test] + fn gpu_backends_are_configured_for_each_host() { + configure_gpu_backends(&mut config(), &target_os("aarch64-apple-darwin")); + configure_gpu_backends(&mut config(), &target_os("x86_64-unknown-linux-gnu")); + } + + #[test] + #[serial] + fn vulkan_linking_covers_every_platform_arm() { + let scratch = ScratchDir::new("cmake-vulkan"); + unsafe { std::env::set_var("VULKAN_SDK", scratch.path()) }; + + configure_vulkan_linking(&mut config(), &target_os("x86_64-pc-windows-msvc")); + configure_vulkan_linking(&mut config(), &target_os("x86_64-unknown-linux-gnu")); + configure_vulkan_linking(&mut config(), &target_os("aarch64-apple-darwin")); + + unsafe { std::env::remove_var("VULKAN_SDK") }; + + configure_vulkan_linking(&mut config(), &target_os("x86_64-unknown-linux-gnu")); + + assert_eq!( + std::env::var("TrackFileAccess").as_deref(), + Ok("false"), + "the windows arm must disable MSBuild file tracking" + ); + + unsafe { std::env::remove_var("TrackFileAccess") }; + } + + #[test] + fn openmp_is_disabled_for_android_and_configured_elsewhere() { + configure_openmp(&mut config(), &target_os("aarch64-linux-android")); + configure_openmp(&mut config(), &target_os("aarch64-apple-darwin")); + } + + #[test] + fn system_ggml_configuration_runs() { + configure_system_ggml(&mut config()); + } + + #[test] + fn the_base_defines_turn_off_upstream_extras() { + configure_base_defines(&mut config()); + } + + /// Drives the whole configure-and-build path against a minimal CMake + /// project, so the orchestration is exercised without building llama.cpp. + #[test] + #[serial] + fn a_minimal_project_is_configured_and_built() { + let scratch = ScratchDir::new("cmake-build"); + let source_dir = scratch.path().join("source"); + std::fs::create_dir_all(&source_dir).expect("source dir must be creatable"); + std::fs::write( + source_dir.join("CMakeLists.txt"), + b"cmake_minimum_required(VERSION 3.14)\n\ + project(probe C)\n\ + add_library(probe STATIC probe.c)\n\ + install(TARGETS probe ARCHIVE DESTINATION lib)\n", + ) + .expect("CMakeLists must be writable"); + std::fs::write( + source_dir.join("probe.c"), + b"int probe(void) { return 1; }\n", + ) + .expect("source must be writable"); + + let context = crate::BuildContext { + manifest_dir: scratch.path().to_path_buf(), + out_dir: scratch.path().join("out"), + target_dir: scratch.path().join("target"), + cmake_dir: scratch.path().join("cmake-out"), + llama_src: source_dir, + target_os: target_os("aarch64-apple-darwin"), + target_triple: "aarch64-apple-darwin".to_owned(), + build_shared_libs: false, + profile: "Release".to_owned(), + static_crt: false, + android_ndk: None, + }; + + let install_dir = + crate::cc_test_environment::with_cc_environment_value(scratch.path(), || { + super::configure_and_build(&context) + }); + + assert!( + install_dir.join("lib").join("libprobe.a").exists(), + "the archive must be installed under the cmake out dir" + ); + } +} diff --git a/llama-cpp-bindings-build/src/cpp_wrapper.rs b/llama-cpp-bindings-build/src/cpp_wrapper.rs index c4a896f55..a66cce547 100644 --- a/llama-cpp-bindings-build/src/cpp_wrapper.rs +++ b/llama-cpp-bindings-build/src/cpp_wrapper.rs @@ -1,45 +1,186 @@ use std::path::Path; +use std::path::PathBuf; use crate::glob_paths; use crate::target_os::TargetOs; +fn msvc_flags(target_os: &TargetOs) -> &'static [&'static str] { + if target_os.is_msvc() { + &["/std:c++17", "/EHsc"] + } else { + &[] + } +} + +fn links_stdlib_statically(target_os: &TargetOs) -> bool { + target_os.is_android() && cfg!(feature = "static-stdcxx") +} + const WRAPPER_SOURCE_PATTERNS: &[&str] = &["wrapper_*.cpp"]; -pub fn compile_cpp_wrappers(llama_src: &Path, target_os: &TargetOs) { - let mut build = cc::Build::new(); +fn wrapper_include_dirs(wrapper_dir: &Path, llama_src: &Path) -> Vec { + vec![ + wrapper_dir.to_path_buf(), + wrapper_dir.join("GSL/include"), + llama_src.to_path_buf(), + llama_src.join("common"), + llama_src.join("include"), + llama_src.join("ggml/include"), + llama_src.join("vendor"), + ] +} - build - .cpp(true) - .warnings(false) - .include(".") - .include("GSL/include") - .include(llama_src) - .include(llama_src.join("common")) - .include(llama_src.join("include")) - .include(llama_src.join("ggml/include")) - .include(llama_src.join("vendor")) - .flag_if_supported("-std=c++17") - .pic(true); +fn wrapper_sources(wrapper_dir: &Path) -> Vec { + let mut sources = Vec::new(); for pattern in WRAPPER_SOURCE_PATTERNS { - match glob_paths::collect_paths(pattern) { - Ok(paths) => { - for path in paths { - build.file(&path); - } - } + let scoped = wrapper_dir.join(pattern); + + match glob_paths::collect_paths(&scoped.to_string_lossy()) { + Ok(paths) => sources.extend(paths), Err(error) => panic!("cpp wrapper discovery failed: {error}"), } } - if target_os.is_msvc() { - build.flag("/std:c++17"); - build.flag("/EHsc"); + sources +} + +pub fn compile_cpp_wrappers(wrapper_dir: &Path, llama_src: &Path, target_os: &TargetOs) { + wrapper_build(wrapper_dir, llama_src, target_os) + .compile("llama_cpp_bindings_sys_common_wrapper"); +} + +fn wrapper_build(wrapper_dir: &Path, llama_src: &Path, target_os: &TargetOs) -> cc::Build { + let mut build = cc::Build::new(); + + build.cpp(true).warnings(false); + + for include_dir in wrapper_include_dirs(wrapper_dir, llama_src) { + build.include(include_dir); } - if target_os.is_android() && cfg!(feature = "static-stdcxx") { + build.flag_if_supported("-std=c++17").pic(true); + + for source in wrapper_sources(wrapper_dir) { + build.file(&source); + } + + for flag in msvc_flags(target_os) { + build.flag(flag); + } + + if links_stdlib_statically(target_os) { build.cpp_link_stdlib(None); } - build.compile("llama_cpp_bindings_sys_common_wrapper"); + build +} + +#[cfg(test)] +mod tests { + use serial_test::serial; + + use crate::cc_test_environment::with_cc_environment; + use crate::scratch_dir::ScratchDir; + use crate::target_os::TargetOs; + + use super::compile_cpp_wrappers; + use super::wrapper_include_dirs; + use super::wrapper_sources; + + #[test] + fn the_include_list_covers_the_wrapper_and_upstream_trees() { + let dirs = wrapper_include_dirs( + std::path::Path::new("/sys"), + std::path::Path::new("/sys/llama.cpp"), + ); + + assert!(dirs.contains(&std::path::PathBuf::from("/sys"))); + assert!(dirs.contains(&std::path::PathBuf::from("/sys/GSL/include"))); + assert!(dirs.contains(&std::path::PathBuf::from("/sys/llama.cpp/ggml/include"))); + } + + #[test] + fn only_wrapper_sources_are_discovered() { + let scratch = ScratchDir::new("cppwrapper-sources"); + std::fs::write(scratch.path().join("wrapper_a.cpp"), b"").expect("source must be writable"); + std::fs::write(scratch.path().join("other.cpp"), b"").expect("source must be writable"); + + let sources = wrapper_sources(scratch.path()); + + assert_eq!(sources.len(), 1, "got: {sources:?}"); + assert!(sources[0].ends_with("wrapper_a.cpp")); + } + + #[test] + #[should_panic(expected = "cpp wrapper discovery failed")] + fn a_directory_without_wrappers_panics() { + let scratch = ScratchDir::new("cppwrapper-empty"); + + let _ = wrapper_sources(scratch.path()); + } + + #[test] + #[serial] + fn wrappers_are_compiled_into_an_archive() { + let scratch = ScratchDir::new("cppwrapper-compile"); + std::fs::write( + scratch.path().join("wrapper_probe.cpp"), + b"extern \"C\" int llama_rs_probe() { return 7; }\n", + ) + .expect("source must be writable"); + + with_cc_environment(scratch.path(), || { + compile_cpp_wrappers( + scratch.path(), + &scratch.path().join("llama.cpp"), + &TargetOs::from_target_triple("aarch64-apple-darwin").expect("supported triple"), + ); + }); + + assert!( + scratch + .path() + .join("libllama_cpp_bindings_sys_common_wrapper.a") + .exists(), + "the wrapper archive must be produced" + ); + } + + #[test] + fn platform_flag_decisions_cover_every_target() { + let msvc = TargetOs::from_target_triple("x86_64-pc-windows-msvc").expect("msvc"); + let android = TargetOs::from_target_triple("aarch64-linux-android").expect("android"); + let apple = TargetOs::from_target_triple("aarch64-apple-darwin").expect("apple"); + + assert_eq!(super::msvc_flags(&msvc), &["/std:c++17", "/EHsc"]); + assert!(super::msvc_flags(&apple).is_empty()); + assert!(super::msvc_flags(&android).is_empty()); + + assert_eq!( + super::links_stdlib_statically(&android), + cfg!(feature = "static-stdcxx") + ); + assert!(!super::links_stdlib_statically(&apple)); + } + + #[test] + fn the_builder_applies_flags_for_every_target_without_compiling() { + let scratch = ScratchDir::new("cppwrapper-builder"); + std::fs::write(scratch.path().join("wrapper_a.cpp"), b"").expect("source must be writable"); + + for triple in [ + "x86_64-pc-windows-msvc", + "aarch64-linux-android", + "aarch64-apple-darwin", + ] { + let target_os = TargetOs::from_target_triple(triple).expect(triple); + + let _ = super::wrapper_build( + scratch.path(), + &scratch.path().join("llama.cpp"), + &target_os, + ); + } + } } diff --git a/llama-cpp-bindings-build/src/cpp_wrapper_mtmd.rs b/llama-cpp-bindings-build/src/cpp_wrapper_mtmd.rs index 92f561e6e..b0e5f9c05 100644 --- a/llama-cpp-bindings-build/src/cpp_wrapper_mtmd.rs +++ b/llama-cpp-bindings-build/src/cpp_wrapper_mtmd.rs @@ -1,55 +1,197 @@ use std::path::Path; +use std::path::PathBuf; use crate::glob_paths; use crate::target_os::TargetOs; +fn msvc_flags(target_os: &TargetOs) -> &'static [&'static str] { + if target_os.is_msvc() { + &["/std:c++17", "/EHsc"] + } else { + &[] + } +} + +fn links_stdlib_statically(target_os: &TargetOs) -> bool { + target_os.is_android() && cfg!(feature = "static-stdcxx") +} + const MTMD_SKIP_FILES: &[&str] = &["mtmd-cli.cpp", "deprecation-warning.cpp"]; +fn mtmd_include_dirs(llama_src: &Path, mtmd_src: &Path) -> Vec { + vec![ + mtmd_src.to_path_buf(), + llama_src.to_path_buf(), + llama_src.join("include"), + llama_src.join("ggml/include"), + llama_src.join("common"), + llama_src.join("vendor"), + ] +} + +fn mtmd_sources(mtmd_src: &Path) -> Vec { + let pattern = mtmd_src.join("**/*.cpp"); + + let paths = match glob_paths::collect_paths(&pattern.to_string_lossy()) { + Ok(paths) => paths, + Err(error) => panic!("mtmd source discovery failed: {error}"), + }; + + paths + .into_iter() + .filter(|path| { + let filename = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + + !MTMD_SKIP_FILES.contains(&filename) + }) + .collect() +} + pub fn compile_mtmd(llama_src: &Path, target_os: &TargetOs) { + mtmd_build(llama_src, target_os).compile("mtmd"); +} + +fn mtmd_build(llama_src: &Path, target_os: &TargetOs) -> cc::Build { let mtmd_src = llama_src.join("tools/mtmd"); let mut build = cc::Build::new(); - build - .cpp(true) - .warnings(false) - .include(&mtmd_src) - .include(llama_src) - .include(llama_src.join("include")) - .include(llama_src.join("ggml/include")) - .include(llama_src.join("common")) - .include(llama_src.join("vendor")) - .flag_if_supported("-std=c++17") - .pic(true); + build.cpp(true).warnings(false); - if target_os.is_msvc() { - build.flag("/std:c++17"); - build.flag("/EHsc"); + for include_dir in mtmd_include_dirs(llama_src, &mtmd_src) { + build.include(include_dir); } - if target_os.is_android() && cfg!(feature = "static-stdcxx") { + build.flag_if_supported("-std=c++17").pic(true); + + for flag in msvc_flags(target_os) { + build.flag(flag); + } + + if links_stdlib_statically(target_os) { build.cpp_link_stdlib(None); } - let pattern = mtmd_src.join("**/*.cpp"); - let pattern_str = pattern.to_string_lossy(); + for source in mtmd_sources(&mtmd_src) { + build.file(&source); + } - let paths = match glob_paths::collect_paths(&pattern_str) { - Ok(paths) => paths, - Err(error) => panic!("mtmd source discovery failed: {error}"), - }; + build +} + +#[cfg(test)] +mod tests { + use serial_test::serial; + + use crate::cc_test_environment::with_cc_environment; + use crate::scratch_dir::ScratchDir; + use crate::target_os::TargetOs; + + use super::compile_mtmd; + use super::mtmd_include_dirs; + use super::mtmd_sources; + + fn llama_src_with_mtmd(scratch: &ScratchDir) -> std::path::PathBuf { + let llama_src = scratch.path().join("llama.cpp"); + let mtmd_src = llama_src.join("tools/mtmd"); + std::fs::create_dir_all(&mtmd_src).expect("mtmd dir must be creatable"); + std::fs::write( + mtmd_src.join("probe.cpp"), + b"extern \"C\" int mtmd_probe() { return 3; }\n", + ) + .expect("source must be writable"); - for path in paths { - let filename = path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(); + llama_src + } + + #[test] + fn the_include_list_covers_mtmd_and_upstream_trees() { + let dirs = mtmd_include_dirs( + std::path::Path::new("/llama"), + std::path::Path::new("/llama/tools/mtmd"), + ); + + assert!(dirs.contains(&std::path::PathBuf::from("/llama/tools/mtmd"))); + assert!(dirs.contains(&std::path::PathBuf::from("/llama/vendor"))); + } - if MTMD_SKIP_FILES.contains(&filename) { - continue; + #[test] + fn the_skip_list_is_excluded_from_discovery() { + let scratch = ScratchDir::new("mtmd-skip"); + let mtmd_src = scratch.path().join("tools/mtmd"); + std::fs::create_dir_all(mtmd_src.join("nested")).expect("dirs must be creatable"); + for name in ["kept.cpp", "mtmd-cli.cpp", "deprecation-warning.cpp"] { + std::fs::write(mtmd_src.join(name), b"").expect("source must be writable"); } + std::fs::write(mtmd_src.join("nested/also-kept.cpp"), b"") + .expect("nested source must be writable"); - build.file(&path); + let sources = mtmd_sources(&mtmd_src); + + assert_eq!(sources.len(), 2, "got: {sources:?}"); + assert!(sources.iter().any(|path| path.ends_with("kept.cpp"))); + assert!(sources.iter().any(|path| path.ends_with("also-kept.cpp"))); } - build.compile("mtmd"); + #[test] + #[should_panic(expected = "mtmd source discovery failed")] + fn a_tree_without_sources_panics() { + let scratch = ScratchDir::new("mtmd-empty"); + + let _ = mtmd_sources(scratch.path()); + } + + #[test] + #[serial] + fn mtmd_sources_are_compiled_into_an_archive() { + let scratch = ScratchDir::new("mtmd-compile"); + let llama_src = llama_src_with_mtmd(&scratch); + + with_cc_environment(scratch.path(), || { + compile_mtmd( + &llama_src, + &TargetOs::from_target_triple("aarch64-apple-darwin").expect("supported triple"), + ); + }); + + assert!( + scratch.path().join("libmtmd.a").exists(), + "the mtmd archive must be produced" + ); + } + + #[test] + fn platform_flag_decisions_cover_every_target() { + let msvc = TargetOs::from_target_triple("x86_64-pc-windows-msvc").expect("msvc"); + let android = TargetOs::from_target_triple("aarch64-linux-android").expect("android"); + let apple = TargetOs::from_target_triple("aarch64-apple-darwin").expect("apple"); + + assert_eq!(super::msvc_flags(&msvc), &["/std:c++17", "/EHsc"]); + assert!(super::msvc_flags(&apple).is_empty()); + assert!(super::msvc_flags(&android).is_empty()); + + assert_eq!( + super::links_stdlib_statically(&android), + cfg!(feature = "static-stdcxx") + ); + assert!(!super::links_stdlib_statically(&apple)); + } + + #[test] + fn the_builder_applies_flags_for_every_target_without_compiling() { + let scratch = ScratchDir::new("mtmd-builder"); + let llama_src = llama_src_with_mtmd(&scratch); + + for triple in [ + "x86_64-pc-windows-msvc", + "aarch64-linux-android", + "aarch64-apple-darwin", + ] { + let target_os = TargetOs::from_target_triple(triple).expect(triple); + + let _ = super::mtmd_build(&llama_src, &target_os); + } + } } diff --git a/llama-cpp-bindings-build/src/glob_paths.rs b/llama-cpp-bindings-build/src/glob_paths.rs index 91da5a095..6ebc15e13 100644 --- a/llama-cpp-bindings-build/src/glob_paths.rs +++ b/llama-cpp-bindings-build/src/glob_paths.rs @@ -47,3 +47,89 @@ pub fn collect_paths(pattern: &str) -> Result, GlobPathsError> { Ok(paths) } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::scratch_dir::ScratchDir; + + use super::GlobPathsError; + use super::collect_paths; + + fn manifest_relative(pattern: &str) -> String { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join(pattern) + .to_string_lossy() + .into_owned() + } + + #[test] + fn matching_pattern_returns_sorted_paths() { + let paths = collect_paths(&manifest_relative("src/*.rs")).expect("sources must match"); + + assert!(paths.len() > 1, "the crate has several source files"); + + let mut sorted = paths.clone(); + sorted.sort(); + + assert_eq!(paths, sorted, "collect_paths must return sorted paths"); + assert!( + paths.iter().any(|path| path.ends_with("glob_paths.rs")), + "this very file must be among the matches" + ); + } + + #[test] + fn an_invalid_pattern_reports_the_pattern() { + let error = collect_paths("a/**b/c").expect_err("malformed recursive glob must fail"); + + assert!(matches!( + error, + GlobPathsError::InvalidPattern { ref pattern, .. } if pattern == "a/**b/c" + )); + assert!(error.to_string().contains("invalid glob pattern")); + } + + #[test] + fn a_pattern_matching_nothing_reports_no_matches() { + let scratch = ScratchDir::new("glob-empty"); + let pattern = scratch + .path() + .join("*.nothing-here") + .to_string_lossy() + .into_owned(); + + let error = collect_paths(&pattern).expect_err("empty directory must fail"); + + assert!(matches!( + error, + GlobPathsError::NoMatches { pattern: ref reported } if *reported == pattern + )); + assert!(error.to_string().contains("no files matched")); + } + + #[cfg(unix)] + #[test] + fn an_unreadable_directory_reports_an_entry_error() { + use std::os::unix::fs::PermissionsExt; + + let scratch = ScratchDir::new("glob-unreadable"); + let locked = scratch.path().join("locked"); + std::fs::create_dir_all(&locked).expect("nested directory must be creatable"); + std::fs::write(locked.join("inner.txt"), b"x").expect("file must be writable"); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)) + .expect("permissions must be settable"); + + let pattern = scratch.path().join("*/*").to_string_lossy().into_owned(); + let outcome = collect_paths(&pattern); + + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)) + .expect("permissions must be restorable for cleanup"); + + let error = outcome.expect_err("an unreadable directory must surface an entry error"); + + assert!(matches!(error, GlobPathsError::EntryError { .. })); + assert!(error.to_string().contains("glob entry failed")); + } +} diff --git a/llama-cpp-bindings-build/src/host_platform.rs b/llama-cpp-bindings-build/src/host_platform.rs new file mode 100644 index 000000000..cf17a91d8 --- /dev/null +++ b/llama-cpp-bindings-build/src/host_platform.rs @@ -0,0 +1,103 @@ +/// The platform whose library-naming conventions apply to a build. +/// +/// Taking this as a parameter rather than branching on `cfg!` inline keeps the +/// naming rules for every platform exercisable from any host. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HostPlatform { + Windows, + MacOs, + Unixlike, +} + +impl HostPlatform { + #[must_use] + pub fn current() -> Self { + Self::from_os(std::env::consts::OS) + } + + /// Resolves from an OS name rather than a `cfg!` branch, so every + /// platform's rules stay reachable from any host. + #[must_use] + pub fn from_os(os: &str) -> Self { + match os { + "windows" => Self::Windows, + "macos" => Self::MacOs, + _ => Self::Unixlike, + } + } + + /// Glob for shared libraries produced by a cmake build. + #[must_use] + pub const fn shared_library_pattern(self) -> &'static str { + match self { + Self::Windows => "*.dll", + Self::MacOs => "*.dylib", + Self::Unixlike => "*.so", + } + } + + /// Directory a cmake install places shared libraries into. + #[must_use] + pub const fn shared_library_dir(self) -> &'static str { + match self { + Self::Windows => "bin", + Self::MacOs | Self::Unixlike => "lib", + } + } + + /// Glob for the libraries the linker should be told about. + #[must_use] + pub const fn link_library_pattern(self, build_shared_libs: bool) -> &'static str { + match self { + Self::Windows => "*.lib", + Self::MacOs if build_shared_libs => "*.dylib", + Self::Unixlike if build_shared_libs => "*.so", + Self::MacOs | Self::Unixlike => "*.a", + } + } +} + +#[cfg(test)] +mod tests { + use super::HostPlatform; + + #[test] + fn every_os_name_resolves_to_its_platform() { + assert_eq!(HostPlatform::from_os("windows"), HostPlatform::Windows); + assert_eq!(HostPlatform::from_os("macos"), HostPlatform::MacOs); + assert_eq!(HostPlatform::from_os("linux"), HostPlatform::Unixlike); + assert_eq!(HostPlatform::from_os("freebsd"), HostPlatform::Unixlike); + } + + #[test] + fn the_current_platform_resolves_from_the_running_os() { + assert_eq!( + HostPlatform::current(), + HostPlatform::from_os(std::env::consts::OS) + ); + } + + #[test] + fn every_platform_names_its_shared_libraries() { + assert_eq!(HostPlatform::Windows.shared_library_pattern(), "*.dll"); + assert_eq!(HostPlatform::MacOs.shared_library_pattern(), "*.dylib"); + assert_eq!(HostPlatform::Unixlike.shared_library_pattern(), "*.so"); + } + + #[test] + fn only_windows_installs_shared_libraries_beside_the_binaries() { + assert_eq!(HostPlatform::Windows.shared_library_dir(), "bin"); + assert_eq!(HostPlatform::MacOs.shared_library_dir(), "lib"); + assert_eq!(HostPlatform::Unixlike.shared_library_dir(), "lib"); + } + + #[test] + fn link_patterns_follow_the_platform_and_link_kind() { + assert_eq!(HostPlatform::Windows.link_library_pattern(false), "*.lib"); + assert_eq!(HostPlatform::Windows.link_library_pattern(true), "*.lib"); + assert_eq!(HostPlatform::MacOs.link_library_pattern(false), "*.a"); + assert_eq!(HostPlatform::MacOs.link_library_pattern(true), "*.dylib"); + assert_eq!(HostPlatform::Unixlike.link_library_pattern(false), "*.a"); + assert_eq!(HostPlatform::Unixlike.link_library_pattern(true), "*.so"); + } +} diff --git a/llama-cpp-bindings-build/src/lib.rs b/llama-cpp-bindings-build/src/lib.rs index a8645dfdf..9c7248c24 100644 --- a/llama-cpp-bindings-build/src/lib.rs +++ b/llama-cpp-bindings-build/src/lib.rs @@ -1,13 +1,18 @@ mod android_ndk; mod bindgen_config; +#[cfg(test)] +mod cc_test_environment; mod cmake_config; mod cpp_wrapper; mod cpp_wrapper_mtmd; mod glob_paths; +mod host_platform; mod library_asset_extraction; mod library_linking; mod library_name_extraction; mod rebuild_tracking; +#[cfg(test)] +mod scratch_dir; mod shared_libs; mod stable_cmake_build_dir; mod target_os; @@ -30,6 +35,7 @@ macro_rules! debug_log { #[derive(Debug)] pub struct BuildContext { + pub manifest_dir: PathBuf, pub out_dir: PathBuf, pub target_dir: PathBuf, pub cmake_dir: PathBuf, @@ -54,7 +60,8 @@ impl BuildContext { let target_dir = cargo_target_dir(&out_dir); let manifest_dir = env::var("CARGO_MANIFEST_DIR") .expect("CARGO_MANIFEST_DIR env var is required in build scripts"); - let llama_src = Path::new(&manifest_dir).join("llama.cpp"); + let manifest_dir = PathBuf::from(manifest_dir); + let llama_src = manifest_dir.join("llama.cpp"); let build_shared_libs = env::var("LLAMA_BUILD_SHARED_LIBS") .map_or_else(|_| cfg!(feature = "dynamic-link"), |value| value == "1"); @@ -83,13 +90,14 @@ impl BuildContext { ); debug_log!("TARGET: {}", target_triple); - debug_log!("CARGO_MANIFEST_DIR: {}", manifest_dir); + debug_log!("CARGO_MANIFEST_DIR: {}", manifest_dir.display()); debug_log!("TARGET_DIR: {}", target_dir.display()); debug_log!("OUT_DIR: {}", out_dir.display()); debug_log!("CMAKE_DIR: {}", cmake_dir.display()); debug_log!("BUILD_SHARED: {}", build_shared_libs); Self { + manifest_dir, out_dir, target_dir, cmake_dir, @@ -115,9 +123,10 @@ fn cargo_target_dir(out_dir: &Path) -> PathBuf { pub fn build() { let context = BuildContext::detect(); - rebuild_tracking::register_rebuild_triggers(&context.llama_src); + rebuild_tracking::register_rebuild_triggers(&context.manifest_dir, &context.llama_src); bindgen_config::generate_bindings( + &context.manifest_dir, &context.llama_src, &context.out_dir, &context.target_os, @@ -125,7 +134,11 @@ pub fn build() { context.android_ndk.as_ref(), ); - cpp_wrapper::compile_cpp_wrappers(&context.llama_src, &context.target_os); + cpp_wrapper::compile_cpp_wrappers( + &context.manifest_dir, + &context.llama_src, + &context.target_os, + ); let build_dir = cmake_config::configure_and_build(&context); @@ -144,3 +157,220 @@ pub fn build() { shared_libs::copy_shared_libraries(&context.cmake_dir, &context.target_dir); } } + +#[cfg(test)] +mod tests { + use std::path::Path; + + use serial_test::serial; + + use crate::scratch_dir::ScratchDir; + use crate::target_os::TargetOs; + + use super::BuildContext; + use super::cargo_target_dir; + + /// A complete stand-in for the sys crate: wrapper headers and sources, and + /// a minimal CMake project in place of llama.cpp. Lets `build()` run end to + /// end in about a second instead of compiling upstream. + fn synthetic_sys_crate(scratch: &ScratchDir) -> std::path::PathBuf { + let manifest_dir = scratch.path().join("sys"); + let llama_src = manifest_dir.join("llama.cpp"); + + for relative in ["include", "ggml/include", "common", "vendor", "tools/mtmd"] { + std::fs::create_dir_all(llama_src.join(relative)).expect("tree must be creatable"); + } + std::fs::create_dir_all(manifest_dir.join("GSL/include")).expect("gsl must be creatable"); + + std::fs::write(manifest_dir.join("wrapper.h"), b"int llama_probe(void);\n") + .expect("header must be writable"); + std::fs::write( + manifest_dir.join("wrapper_mtmd.h"), + b"int mtmd_probe(void);\n", + ) + .expect("header must be writable"); + std::fs::write( + manifest_dir.join("wrapper_probe.cpp"), + b"extern \"C\" int llama_probe() { return 1; }\n", + ) + .expect("wrapper source must be writable"); + std::fs::write( + llama_src.join("tools/mtmd/probe.cpp"), + b"extern \"C\" int mtmd_probe() { return 2; }\n", + ) + .expect("mtmd source must be writable"); + std::fs::write( + llama_src.join("CMakeLists.txt"), + b"cmake_minimum_required(VERSION 3.14)\n\ + project(probe C)\n\ + add_library(probe probe.c)\n\ + install(TARGETS probe ARCHIVE DESTINATION lib LIBRARY DESTINATION lib)\n", + ) + .expect("CMakeLists must be writable"); + std::fs::write( + llama_src.join("probe.c"), + b"int probe(void) { return 1; }\n", + ) + .expect("cmake source must be writable"); + + manifest_dir + } + + fn run_build_against(manifest_dir: &Path, shared_libs: &str) { + let out_dir = manifest_dir.join("target/debug/build/sys-abc/out"); + std::fs::create_dir_all(&out_dir).expect("out dir must be creatable"); + let rocm_root = manifest_dir.join("rocm"); + std::fs::create_dir_all(rocm_root.join("lib")).expect("rocm lib must be creatable"); + let host = format!("{}-apple-darwin", std::env::consts::ARCH); + + unsafe { + std::env::set_var("ROCM_PATH", &rocm_root); + std::env::set_var("VULKAN_SDK", manifest_dir); + std::env::set_var("CARGO_MANIFEST_DIR", manifest_dir); + std::env::set_var("OUT_DIR", &out_dir); + std::env::set_var("TARGET", &host); + std::env::set_var("HOST", &host); + std::env::set_var("OPT_LEVEL", "0"); + std::env::set_var("PROFILE", "debug"); + std::env::set_var("NUM_JOBS", "1"); + std::env::set_var("LLAMA_BUILD_SHARED_LIBS", shared_libs); + std::env::remove_var("LLAMA_CMAKE_BUILD_DIR_OVERRIDE"); + std::env::remove_var("LLAMA_LIB_PROFILE"); + std::env::remove_var("LLAMA_STATIC_CRT"); + } + + super::build(); + + unsafe { + std::env::remove_var("LLAMA_BUILD_SHARED_LIBS"); + std::env::remove_var("OPT_LEVEL"); + std::env::remove_var("PROFILE"); + std::env::remove_var("NUM_JOBS"); + std::env::remove_var("ROCM_PATH"); + std::env::remove_var("VULKAN_SDK"); + std::env::remove_var("TrackFileAccess"); + } + } + + #[test] + #[serial] + fn a_static_build_runs_every_stage() { + let scratch = ScratchDir::new("build-static"); + + run_build_against(&synthetic_sys_crate(&scratch), "0"); + } + + #[test] + #[serial] + fn a_shared_build_also_copies_the_libraries() { + let scratch = ScratchDir::new("build-shared"); + + run_build_against(&synthetic_sys_crate(&scratch), "1"); + } + + #[test] + fn the_target_dir_is_three_levels_above_out_dir() { + let out_dir = Path::new("/w/target/debug/build/llama-cpp-bindings-sys-abc123/out"); + + assert_eq!(cargo_target_dir(out_dir), Path::new("/w/target/debug")); + } + + #[test] + #[should_panic(expected = "OUT_DIR is not deep enough")] + fn a_shallow_out_dir_panics() { + let _ = cargo_target_dir(Path::new("/out")); + } + + #[test] + #[serial] + fn detect_reads_the_build_script_environment() { + let scratch = ScratchDir::new("context-detect"); + let out_dir = scratch.path().join("target/debug/build/crate-abc/out"); + std::fs::create_dir_all(&out_dir).expect("out dir must be creatable"); + + unsafe { + std::env::set_var("TARGET", "aarch64-apple-darwin"); + std::env::set_var("OUT_DIR", &out_dir); + std::env::set_var("CARGO_MANIFEST_DIR", scratch.path()); + std::env::remove_var("LLAMA_CMAKE_BUILD_DIR_OVERRIDE"); + std::env::remove_var("LLAMA_BUILD_SHARED_LIBS"); + std::env::remove_var("LLAMA_LIB_PROFILE"); + std::env::remove_var("LLAMA_STATIC_CRT"); + } + + let context = BuildContext::detect(); + + assert!(matches!(context.target_os, TargetOs::Apple(_))); + assert_eq!(context.target_triple, "aarch64-apple-darwin"); + assert_eq!(context.manifest_dir, scratch.path()); + assert_eq!(context.llama_src, scratch.path().join("llama.cpp")); + assert_eq!(context.target_dir, scratch.path().join("target/debug")); + assert_eq!(context.profile, "Release"); + assert!(!context.static_crt); + assert_eq!( + context.build_shared_libs, + cfg!(feature = "dynamic-link"), + "with no override, shared linking follows the dynamic-link feature" + ); + assert!(context.cmake_dir.is_dir()); + assert!(context.android_ndk.is_none()); + assert!(format!("{context:?}").contains("BuildContext")); + } + + #[test] + #[serial] + fn detect_honours_the_profile_and_crt_overrides() { + let scratch = ScratchDir::new("context-overrides"); + let out_dir = scratch.path().join("target/debug/build/crate-abc/out"); + std::fs::create_dir_all(&out_dir).expect("out dir must be creatable"); + + unsafe { + std::env::set_var("TARGET", "x86_64-unknown-linux-gnu"); + std::env::set_var("OUT_DIR", &out_dir); + std::env::set_var("CARGO_MANIFEST_DIR", scratch.path()); + std::env::set_var("LLAMA_LIB_PROFILE", "Debug"); + std::env::set_var("LLAMA_STATIC_CRT", "1"); + std::env::set_var("LLAMA_BUILD_SHARED_LIBS", "1"); + } + + let context = BuildContext::detect(); + + unsafe { + std::env::remove_var("LLAMA_LIB_PROFILE"); + std::env::remove_var("LLAMA_STATIC_CRT"); + std::env::remove_var("LLAMA_BUILD_SHARED_LIBS"); + } + + assert_eq!(context.profile, "Debug"); + assert!(context.static_crt); + assert!(context.build_shared_libs); + assert!(matches!(context.target_os, TargetOs::Linux)); + } + + #[test] + #[serial] + #[should_panic(expected = "Android NDK detection failed")] + fn an_android_target_without_an_ndk_panics() { + let scratch = ScratchDir::new("context-android"); + let out_dir = scratch.path().join("target/debug/build/crate-abc/out"); + std::fs::create_dir_all(&out_dir).expect("out dir must be creatable"); + + unsafe { + std::env::set_var("TARGET", "aarch64-linux-android"); + std::env::set_var("OUT_DIR", &out_dir); + std::env::set_var("CARGO_MANIFEST_DIR", scratch.path()); + for name in [ + "ANDROID_NDK", + "ANDROID_NDK_ROOT", + "NDK_ROOT", + "CARGO_NDK_ANDROID_NDK", + "ANDROID_SDK_ROOT", + ] { + std::env::remove_var(name); + } + std::env::set_var("ANDROID_HOME", scratch.path()); + } + + let _ = BuildContext::detect(); + } +} diff --git a/llama-cpp-bindings-build/src/library_asset_extraction.rs b/llama-cpp-bindings-build/src/library_asset_extraction.rs index 358f32484..ed98b9dad 100644 --- a/llama-cpp-bindings-build/src/library_asset_extraction.rs +++ b/llama-cpp-bindings-build/src/library_asset_extraction.rs @@ -3,19 +3,15 @@ use std::path::{Path, PathBuf}; use glob::glob; use crate::debug_log; +use crate::host_platform::HostPlatform; pub fn extract_lib_assets(cmake_dir: &Path) -> Vec { - let shared_lib_pattern = if cfg!(windows) { - "*.dll" - } else if cfg!(target_os = "macos") { - "*.dylib" - } else { - "*.so" - }; + extract_lib_assets_for(cmake_dir, HostPlatform::current()) +} - let shared_libs_dir = if cfg!(windows) { "bin" } else { "lib" }; - let libs_dir = cmake_dir.join(shared_libs_dir); - let pattern = libs_dir.join(shared_lib_pattern); +fn extract_lib_assets_for(cmake_dir: &Path, platform: HostPlatform) -> Vec { + let libs_dir = cmake_dir.join(platform.shared_library_dir()); + let pattern = libs_dir.join(platform.shared_library_pattern()); debug_log!("Extract lib assets {}", pattern.display()); let pattern_str = pattern.to_string_lossy(); @@ -36,3 +32,102 @@ pub fn extract_lib_assets(cmake_dir: &Path) -> Vec { files } + +#[cfg(test)] +mod tests { + use crate::scratch_dir::ScratchDir; + + use crate::host_platform::HostPlatform; + + use super::extract_lib_assets; + use super::extract_lib_assets_for; + + fn shared_lib_name(stem: &str) -> String { + HostPlatform::current() + .shared_library_pattern() + .replace('*', stem) + } + + fn assets_dir_name() -> &'static str { + HostPlatform::current().shared_library_dir() + } + + #[test] + fn shared_libraries_in_the_assets_directory_are_collected() { + let scratch = ScratchDir::new("assets-present"); + let assets_dir = scratch.path().join(assets_dir_name()); + std::fs::create_dir_all(&assets_dir).expect("assets directory must be creatable"); + std::fs::write(assets_dir.join(shared_lib_name("libggml")), b"x") + .expect("shared library must be writable"); + std::fs::write(assets_dir.join("notes.txt"), b"x").expect("text file must be writable"); + + let found = extract_lib_assets(scratch.path()); + + assert_eq!(found.len(), 1, "only the shared library matches: {found:?}"); + assert!( + found[0].ends_with(shared_lib_name("libggml")), + "got: {found:?}" + ); + } + + #[test] + fn a_missing_assets_directory_yields_no_files() { + let scratch = ScratchDir::new("assets-absent"); + + assert!(extract_lib_assets(scratch.path()).is_empty()); + } + + #[cfg(unix)] + #[test] + fn an_unreadable_assets_directory_is_reported_and_skipped() { + use std::os::unix::fs::PermissionsExt; + + let scratch = ScratchDir::new("assets-unreadable"); + let assets_dir = scratch.path().join(assets_dir_name()); + std::fs::create_dir_all(&assets_dir).expect("assets directory must be creatable"); + std::fs::write(assets_dir.join(shared_lib_name("libggml")), b"x") + .expect("shared library must be writable"); + std::fs::set_permissions(&assets_dir, std::fs::Permissions::from_mode(0o000)) + .expect("permissions must be settable"); + + let found = extract_lib_assets(scratch.path()); + + std::fs::set_permissions(&assets_dir, std::fs::Permissions::from_mode(0o755)) + .expect("permissions must be restorable for cleanup"); + + assert!( + found.is_empty(), + "unreadable entries are skipped: {found:?}" + ); + } + + #[test] + fn every_platform_collects_its_own_shared_library_flavour() { + for platform in [ + HostPlatform::Windows, + HostPlatform::MacOs, + HostPlatform::Unixlike, + ] { + let scratch = ScratchDir::new("assets-platform"); + let assets_dir = scratch.path().join(platform.shared_library_dir()); + std::fs::create_dir_all(&assets_dir).expect("assets dir must be creatable"); + let name = platform.shared_library_pattern().replace('*', "libggml"); + std::fs::write(assets_dir.join(&name), b"x").expect("library must be writable"); + + let found = extract_lib_assets_for(scratch.path(), platform); + + assert_eq!(found.len(), 1, "{platform:?} -> {found:?}"); + assert!(found[0].ends_with(&name)); + } + } + + #[test] + fn an_invalid_glob_pattern_is_reported_and_yields_no_files() { + let scratch = ScratchDir::new("assets-badglob"); + let cmake_dir = scratch.path().join("a**b"); + std::fs::create_dir_all(cmake_dir.join(assets_dir_name())) + .expect("assets dir must be creatable"); + + assert!(extract_lib_assets(&cmake_dir).is_empty()); + } +} diff --git a/llama-cpp-bindings-build/src/library_linking.rs b/llama-cpp-bindings-build/src/library_linking.rs index d834e9c2c..33c13dd79 100644 --- a/llama-cpp-bindings-build/src/library_linking.rs +++ b/llama-cpp-bindings-build/src/library_linking.rs @@ -2,6 +2,7 @@ use std::env; use std::path::Path; use crate::debug_log; +use crate::host_platform::HostPlatform; use crate::library_name_extraction::extract_lib_names; use crate::target_os::{AppleVariant, TargetOs, WindowsVariant}; @@ -132,11 +133,19 @@ fn link_cuda_libraries(build_shared_libs: bool) { println!("cargo:rerun-if-env-changed=CUDA_PATH"); - for lib_dir in find_cuda_helper::find_cuda_lib_dirs() { + emit_cuda_search_paths(find_cuda_helper::find_cuda_lib_dirs()); + + link_cuda_for(HostPlatform::current()); +} + +fn emit_cuda_search_paths(lib_dirs: impl IntoIterator) { + for lib_dir in lib_dirs { println!("cargo:rustc-link-search=native={}", lib_dir.display()); } +} - if cfg!(target_os = "windows") { +fn link_cuda_for(platform: HostPlatform) { + if platform == HostPlatform::Windows { link_cuda_windows(); } else { link_cuda_unix(); @@ -175,13 +184,7 @@ fn link_rocm_libraries(build_shared_libs: bool) { let rocm_path = env::var("ROCM_PATH") .or_else(|_| env::var("HIP_PATH")) - .unwrap_or_else(|_| { - if cfg!(target_os = "windows") { - "C:\\Program Files\\AMD\\ROCm".to_string() - } else { - "/opt/rocm".to_string() - } - }); + .unwrap_or_else(|_| default_rocm_path(HostPlatform::current())); let rocm_lib = Path::new(&rocm_path).join("lib"); @@ -199,6 +202,13 @@ fn link_rocm_libraries(build_shared_libs: bool) { println!("cargo:rustc-link-lib=dylib=hipblas"); } +fn default_rocm_path(platform: HostPlatform) -> String { + match platform { + HostPlatform::Windows => "C:\\Program Files\\AMD\\ROCm".to_owned(), + HostPlatform::MacOs | HostPlatform::Unixlike => "/opt/rocm".to_owned(), + } +} + fn link_openmp(target_triple: &str) { if cfg!(feature = "openmp") && target_triple.contains("gnu") { println!("cargo:rustc-link-lib=gomp"); @@ -239,12 +249,20 @@ fn link_msvc_system_libraries() { .unwrap_or_default() .contains("crt-static"); - if cfg!(debug_assertions) { - if crt_static { - println!("cargo:rustc-link-lib=libcmtd"); - } else { - println!("cargo:rustc-link-lib=dylib=msvcrtd"); - } + if let Some(debug_runtime) = msvc_debug_runtime(cfg!(debug_assertions), crt_static) { + println!("{debug_runtime}"); + } +} + +fn msvc_debug_runtime(debug_assertions: bool, crt_static: bool) -> Option<&'static str> { + if !debug_assertions { + return None; + } + + if crt_static { + Some("cargo:rustc-link-lib=libcmtd") + } else { + Some("cargo:rustc-link-lib=dylib=msvcrtd") } } @@ -264,7 +282,11 @@ fn link_apple_frameworks(variant: AppleVariant) { } fn macos_link_search_path() -> Option { - let output = std::process::Command::new("clang") + clang_search_dirs("clang") +} + +fn clang_search_dirs(program: &str) -> Option { + let output = std::process::Command::new(program) .arg("--print-search-dirs") .output() .ok()?; @@ -277,8 +299,10 @@ fn macos_link_search_path() -> Option { return None; } - let stdout = String::from_utf8_lossy(&output.stdout); + parse_clang_search_dirs(&String::from_utf8_lossy(&output.stdout)) +} +fn parse_clang_search_dirs(stdout: &str) -> Option { for line in stdout.lines() { if line.contains("libraries: =") { let path = line.split('=').nth(1)?; @@ -291,3 +315,312 @@ fn macos_link_search_path() -> Option { None } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serial_test::serial; + + use crate::host_platform::HostPlatform; + use crate::scratch_dir::ScratchDir; + use crate::target_os::TargetOs; + + use super::clang_search_dirs; + use super::default_rocm_path; + use super::emit_cuda_search_paths; + use super::emit_search_path_with_profile; + use super::emit_search_paths; + use super::link_cmake_built_libraries; + use super::link_cuda_for; + use super::link_cuda_libraries; + use super::link_libraries; + use super::link_llama_common_internal_libraries; + use super::link_openmp; + use super::link_platform_system_libraries; + use super::link_rocm_libraries; + use super::link_system_ggml_paths; + use super::msvc_debug_runtime; + use super::parse_clang_search_dirs; + + fn archive_name(stem: &str) -> String { + crate::host_platform::HostPlatform::current() + .link_library_pattern(false) + .replace('*', stem) + } + + fn cmake_dir_with_library(scratch: &ScratchDir) -> PathBuf { + let cmake_dir = scratch.path().join("cmake"); + let libs_dir = cmake_dir.join("lib"); + std::fs::create_dir_all(&libs_dir).expect("lib dir must be creatable"); + std::fs::write(libs_dir.join(archive_name("libggml")), b"x") + .expect("archive must be writable"); + + cmake_dir + } + + #[test] + fn search_paths_cover_lib_lib64_and_the_build_dir() { + let scratch = ScratchDir::new("linking-search"); + + emit_search_paths(&scratch.path().join("cmake"), scratch.path()); + } + + #[test] + fn system_ggml_paths_are_skipped_without_the_feature_or_a_cache() { + let scratch = ScratchDir::new("linking-ggml"); + + link_system_ggml_paths(scratch.path()); + } + + #[test] + fn cmake_built_libraries_are_linked() { + let scratch = ScratchDir::new("linking-built"); + let cmake_dir = cmake_dir_with_library(&scratch); + + link_cmake_built_libraries(&cmake_dir, false, "Release"); + } + + #[test] + #[should_panic(expected = "no libraries found in build output")] + fn an_empty_build_output_panics() { + let scratch = ScratchDir::new("linking-empty"); + + link_cmake_built_libraries(scratch.path(), false, "Release"); + } + + #[test] + fn internal_libraries_are_linked_when_their_directories_exist() { + let scratch = ScratchDir::new("linking-internal"); + let cmake_dir = scratch.path().join("cmake"); + std::fs::create_dir_all(cmake_dir.join("build/common/Release")) + .expect("common dir must be creatable"); + std::fs::create_dir_all(cmake_dir.join("build/vendor/cpp-httplib")) + .expect("httplib dir must be creatable"); + + link_llama_common_internal_libraries(&cmake_dir, "Release"); + } + + #[test] + fn internal_libraries_are_skipped_when_absent() { + let scratch = ScratchDir::new("linking-internal-absent"); + + link_llama_common_internal_libraries(scratch.path(), "Release"); + } + + #[test] + fn a_profile_subdirectory_adds_a_second_search_path() { + let scratch = ScratchDir::new("linking-profile"); + std::fs::create_dir_all(scratch.path().join("Release")) + .expect("profile dir must be creatable"); + + emit_search_path_with_profile(scratch.path(), "Release"); + emit_search_path_with_profile(scratch.path(), "Debug"); + } + + /// `link_rocm_libraries` asserts the ROCm lib directory exists, so the + /// environment is pointed at a real one for the duration of the call. + fn with_rocm_root(body: TBody) { + let scratch = ScratchDir::new("linking-rocm"); + std::fs::create_dir_all(scratch.path().join("lib")).expect("rocm lib must be creatable"); + unsafe { std::env::set_var("ROCM_PATH", scratch.path()) }; + + body(); + + unsafe { std::env::remove_var("ROCM_PATH") }; + } + + #[test] + #[serial] + fn accelerator_linking_is_skipped_for_shared_library_builds() { + link_cuda_libraries(true); + with_rocm_root(|| link_rocm_libraries(true)); + } + + /// The accelerator and platform emitters are compiled on every host even + /// though `cfg!` gates keep them unreachable here, so they are driven + /// directly rather than left unexercised. + #[test] + fn cuda_emitters_produce_both_link_flavours() { + super::link_cuda_windows(); + super::link_cuda_unix(); + } + + #[test] + fn the_system_ggml_emitter_covers_both_link_kinds() { + super::link_system_ggml_libraries("static"); + super::link_system_ggml_libraries("dylib"); + } + + #[test] + fn the_msvc_emitter_runs_without_a_windows_host() { + super::link_msvc_system_libraries(); + } + + #[test] + fn the_android_stdlib_emitter_runs_without_an_android_host() { + super::link_android_cpp_stdlib(); + } + + #[test] + fn apple_frameworks_are_emitted_for_both_variants() { + super::link_apple_frameworks(crate::target_os::AppleVariant::MacOS); + super::link_apple_frameworks(crate::target_os::AppleVariant::Other); + } + + #[test] + #[serial] + fn accelerator_linking_runs_for_static_builds() { + link_cuda_libraries(false); + with_rocm_root(|| link_rocm_libraries(false)); + } + + #[test] + fn openmp_is_only_linked_for_gnu_triples() { + link_openmp("x86_64-unknown-linux-gnu"); + link_openmp("aarch64-apple-darwin"); + } + + #[test] + fn every_platform_arm_emits_without_panicking() { + for triple in [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "x86_64-unknown-linux-gnu", + "aarch64-linux-android", + "x86_64-pc-windows-msvc", + "x86_64-pc-windows-gnu", + ] { + let target_os = TargetOs::from_target_triple(triple).expect("supported triple"); + + link_platform_system_libraries(&target_os); + } + } + + #[cfg(target_os = "macos")] + #[test] + #[serial] + fn the_macos_link_search_path_is_discovered_from_clang() { + use super::macos_link_search_path; + + let path = macos_link_search_path().expect("clang ships with the macOS toolchain"); + + assert!(path.ends_with("/lib/darwin"), "got: {path}"); + } + + #[test] + #[serial] + fn linking_end_to_end_emits_every_group() { + let scratch = ScratchDir::new("linking-end-to-end"); + let cmake_dir = cmake_dir_with_library(&scratch); + let target_os = + TargetOs::from_target_triple("aarch64-apple-darwin").expect("supported triple"); + + with_rocm_root(|| { + link_libraries( + &cmake_dir, + scratch.path(), + &target_os, + "aarch64-apple-darwin", + false, + "Release", + ); + }); + } + + #[test] + fn clang_search_dirs_are_parsed_into_a_darwin_lib_path() { + let stdout = "programs: =/usr/bin\nlibraries: =/opt/clang\n"; + + assert_eq!( + parse_clang_search_dirs(stdout), + Some("/opt/clang/lib/darwin".to_owned()) + ); + } + + #[test] + fn clang_output_without_a_libraries_line_yields_nothing() { + assert_eq!(parse_clang_search_dirs("programs: =/usr/bin\n"), None); + assert_eq!(parse_clang_search_dirs(""), None); + } + + #[test] + fn the_msvc_debug_runtime_depends_on_assertions_and_the_crt_kind() { + assert_eq!( + msvc_debug_runtime(true, true), + Some("cargo:rustc-link-lib=libcmtd") + ); + assert_eq!( + msvc_debug_runtime(true, false), + Some("cargo:rustc-link-lib=dylib=msvcrtd") + ); + assert_eq!(msvc_debug_runtime(false, true), None); + assert_eq!(msvc_debug_runtime(false, false), None); + } + + #[test] + fn a_cmake_cache_contributes_system_ggml_search_paths() { + let scratch = ScratchDir::new("linking-ggml-cache"); + let build_dir = scratch.path().join("build"); + std::fs::create_dir_all(&build_dir).expect("build dir must be creatable"); + std::fs::write( + build_dir.join("CMakeCache.txt"), + b"GGML_LIBRARY:FILEPATH=/opt/ggml/lib/libggml.so\n\ + GGML_BASE_LIBRARY:FILEPATH=/opt/ggml/lib/libggml-base.so\n\ + GGML_CPU_LIBRARY:FILEPATH=/opt/ggml/lib/libggml-cpu.so\n\ + UNRELATED:STRING=value\n", + ) + .expect("cache must be writable"); + + link_system_ggml_paths(scratch.path()); + } + + #[test] + fn cuda_linking_dispatches_per_platform() { + link_cuda_for(HostPlatform::Windows); + link_cuda_for(HostPlatform::MacOs); + link_cuda_for(HostPlatform::Unixlike); + } + + #[test] + fn the_default_rocm_path_follows_the_platform() { + assert!(default_rocm_path(HostPlatform::Windows).contains("AMD")); + assert_eq!(default_rocm_path(HostPlatform::MacOs), "/opt/rocm"); + assert_eq!(default_rocm_path(HostPlatform::Unixlike), "/opt/rocm"); + } + + #[test] + fn a_missing_or_failing_clang_yields_no_search_path() { + assert_eq!( + clang_search_dirs("definitely-not-a-real-clang-binary"), + None + ); + assert_eq!( + clang_search_dirs("false"), + None, + "a non-zero exit must be reported and skipped" + ); + } + + #[test] + fn cuda_search_paths_are_emitted_for_each_discovered_directory() { + emit_cuda_search_paths(vec![ + std::path::PathBuf::from("/usr/local/cuda/lib64"), + std::path::PathBuf::from("/opt/cuda/lib"), + ]); + emit_cuda_search_paths(Vec::new()); + } + + #[cfg(feature = "rocm")] + #[test] + #[serial] + #[should_panic(expected = "ROCm libraries not found")] + fn a_missing_rocm_installation_is_reported() { + unsafe { + std::env::set_var("ROCM_PATH", "/definitely/not/a/rocm/installation"); + std::env::remove_var("HIP_PATH"); + } + + link_rocm_libraries(false); + } +} diff --git a/llama-cpp-bindings-build/src/library_name_extraction.rs b/llama-cpp-bindings-build/src/library_name_extraction.rs index b76b6be62..badc3f399 100644 --- a/llama-cpp-bindings-build/src/library_name_extraction.rs +++ b/llama-cpp-bindings-build/src/library_name_extraction.rs @@ -3,6 +3,7 @@ use std::path::Path; use glob::glob; use crate::debug_log; +use crate::host_platform::HostPlatform; fn extract_single_lib_name(path: &Path) -> Option { let stem = path.file_stem()?.to_str()?; @@ -11,10 +12,8 @@ fn extract_single_lib_name(path: &Path) -> Option { return Some(stripped.to_string()); } - if path.extension() == Some(std::ffi::OsStr::new("a")) - && let Some(parent) = path.parent() - { - let renamed_path = parent.join(format!("lib{stem}.a")); + if path.extension() == Some(std::ffi::OsStr::new("a")) { + let renamed_path = path.with_file_name(format!("lib{stem}.a")); if let Err(error) = std::fs::rename(path, &renamed_path) { println!( @@ -29,18 +28,16 @@ fn extract_single_lib_name(path: &Path) -> Option { } pub fn extract_lib_names(cmake_dir: &Path, build_shared_libs: bool) -> Vec { - let lib_pattern = if cfg!(windows) { - "*.lib" - } else if cfg!(target_os = "macos") { - if build_shared_libs { "*.dylib" } else { "*.a" } - } else if build_shared_libs { - "*.so" - } else { - "*.a" - }; + extract_lib_names_for(cmake_dir, build_shared_libs, HostPlatform::current()) +} +fn extract_lib_names_for( + cmake_dir: &Path, + build_shared_libs: bool, + platform: HostPlatform, +) -> Vec { let libs_dir = cmake_dir.join("lib*"); - let pattern = libs_dir.join(lib_pattern); + let pattern = libs_dir.join(platform.link_library_pattern(build_shared_libs)); debug_log!("Extract libs {}", pattern.display()); let pattern_str = pattern.to_string_lossy(); @@ -65,3 +62,165 @@ pub fn extract_lib_names(cmake_dir: &Path, build_shared_libs: bool) -> Vec String { + HostPlatform::current() + .link_library_pattern(build_shared_libs) + .replace('*', stem) + } + + fn cmake_dir_with(scratch: &ScratchDir, file_name: &str) -> PathBuf { + let cmake_dir = scratch.path().join("cmake"); + let libs_dir = cmake_dir.join("lib"); + std::fs::create_dir_all(&libs_dir).expect("lib dir must be creatable"); + std::fs::write(libs_dir.join(file_name), b"x").expect("library must be writable"); + + cmake_dir + } + + #[test] + fn a_lib_prefix_is_stripped_from_the_name() { + let scratch = ScratchDir::new("libname-prefixed"); + let cmake_dir = cmake_dir_with(&scratch, &static_or_shared_name("libggml", false)); + + assert_eq!( + extract_lib_names(&cmake_dir, false), + vec!["ggml".to_owned()] + ); + } + + #[test] + fn shared_libraries_are_matched_when_requested() { + let scratch = ScratchDir::new("libname-shared"); + let cmake_dir = cmake_dir_with(&scratch, &static_or_shared_name("libggml", true)); + + assert_eq!(extract_lib_names(&cmake_dir, true), vec!["ggml".to_owned()]); + } + + #[test] + fn an_unprefixed_archive_is_renamed_and_still_reported() { + let scratch = ScratchDir::new("libname-rename"); + let cmake_dir = cmake_dir_with(&scratch, "ggml.a"); + + let names = extract_lib_names_for(&cmake_dir, false, HostPlatform::Unixlike); + + assert_eq!(names, vec!["ggml".to_owned()]); + assert!( + cmake_dir.join("lib").join("libggml.a").exists(), + "the archive must be renamed so the linker can find it" + ); + } + + #[test] + fn a_path_without_a_file_stem_yields_no_name() { + assert_eq!(extract_single_lib_name(Path::new("..")), None); + } + + #[test] + fn a_missing_cmake_directory_yields_no_names() { + let scratch = ScratchDir::new("libname-absent"); + + assert!(extract_lib_names(scratch.path(), false).is_empty()); + } + + #[test] + fn an_invalid_glob_pattern_is_reported_and_yields_no_names() { + let scratch = ScratchDir::new("libname-badglob"); + let cmake_dir = scratch.path().join("a**b"); + std::fs::create_dir_all(&cmake_dir).expect("directory must be creatable"); + + assert!(extract_lib_names(&cmake_dir, false).is_empty()); + } + + #[test] + fn a_failing_rename_is_reported_and_the_name_is_still_returned() { + let scratch = ScratchDir::new("libname-rename-fails"); + let libs_dir = scratch.path().join("lib"); + std::fs::create_dir_all(&libs_dir).expect("lib dir must be creatable"); + let archive = libs_dir.join("ggml.a"); + std::fs::write(&archive, b"x").expect("archive must be writable"); + // Renaming onto a non-empty directory cannot succeed. + std::fs::create_dir_all(libs_dir.join("libggml.a")).expect("blocker must be creatable"); + std::fs::write(libs_dir.join("libggml.a/occupied"), b"x").expect("blocker must be filled"); + + let name = extract_single_lib_name(&archive); + + assert_eq!(name, Some("ggml".to_owned())); + assert!(archive.exists(), "the archive must be left where it was"); + } + + #[test] + fn every_platform_and_link_kind_finds_its_libraries() { + for platform in [ + HostPlatform::Windows, + HostPlatform::MacOs, + HostPlatform::Unixlike, + ] { + for build_shared_libs in [false, true] { + let scratch = ScratchDir::new("libname-platform"); + let libs_dir = scratch.path().join("lib"); + std::fs::create_dir_all(&libs_dir).expect("lib dir must be creatable"); + let name = platform + .link_library_pattern(build_shared_libs) + .replace('*', "libggml"); + std::fs::write(libs_dir.join(&name), b"x").expect("library must be writable"); + + let names = extract_lib_names_for(scratch.path(), build_shared_libs, platform); + + assert_eq!( + names, + vec!["ggml".to_owned()], + "{platform:?}/{build_shared_libs}" + ); + } + } + } + + #[cfg(unix)] + #[test] + fn an_unreadable_library_directory_is_reported_and_skipped() { + use std::os::unix::fs::PermissionsExt; + + let scratch = ScratchDir::new("libname-unreadable"); + let cmake_dir = scratch.path().join("cmake"); + let libs_dir = cmake_dir.join("lib"); + std::fs::create_dir_all(libs_dir.join("nested")).expect("dirs must be creatable"); + std::fs::write(libs_dir.join("nested/libggml.a"), b"x").expect("archive must be writable"); + std::fs::set_permissions(&libs_dir, std::fs::Permissions::from_mode(0o000)) + .expect("permissions must be settable"); + + let names = extract_lib_names_for(&cmake_dir, false, HostPlatform::Unixlike); + + std::fs::set_permissions(&libs_dir, std::fs::Permissions::from_mode(0o755)) + .expect("permissions must be restorable for cleanup"); + + assert!( + names.is_empty(), + "unreadable entries are skipped: {names:?}" + ); + } + + #[test] + fn an_unprefixed_non_archive_is_reported_without_renaming() { + let name = extract_single_lib_name(Path::new("/build/lib/ggml.dylib")); + + assert_eq!( + name, + Some("ggml".to_owned()), + "only .a archives are renamed; other kinds are reported as-is" + ); + } +} diff --git a/llama-cpp-bindings-build/src/rebuild_tracking.rs b/llama-cpp-bindings-build/src/rebuild_tracking.rs index 6a5c6f77c..50d00c6b8 100644 --- a/llama-cpp-bindings-build/src/rebuild_tracking.rs +++ b/llama-cpp-bindings-build/src/rebuild_tracking.rs @@ -20,11 +20,14 @@ fn is_cmake_file(entry: &DirEntry) -> bool { .is_some_and(|name| name.starts_with("CMake")) } -pub fn register_rebuild_triggers(llama_src: &Path) { +pub fn register_rebuild_triggers(wrapper_dir: &Path, llama_src: &Path) { println!("cargo:rerun-if-changed=build.rs"); for pattern in WRAPPER_TRACKING_PATTERNS { - match glob_paths::collect_paths(pattern) { + let scoped = wrapper_dir.join(pattern); + let scoped = scoped.to_string_lossy(); + + match glob_paths::collect_paths(&scoped) { Ok(paths) => { for path in paths { println!("cargo:rerun-if-changed={}", path.display()); @@ -62,3 +65,119 @@ pub fn register_rebuild_triggers(llama_src: &Path) { } } } + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::path::PathBuf; + + use crate::scratch_dir::ScratchDir; + + use super::is_cmake_file; + use super::is_hidden; + use super::register_rebuild_triggers; + + fn entry_for(path: &Path) -> walkdir::DirEntry { + walkdir::WalkDir::new(path) + .min_depth(1) + .max_depth(1) + .into_iter() + .next() + .expect("the directory must contain one entry") + .expect("the entry must be readable") + } + + fn wrapper_dir_with_sources(scratch: &ScratchDir) -> PathBuf { + let wrapper_dir = scratch.path().join("sys"); + std::fs::create_dir_all(&wrapper_dir).expect("wrapper dir must be creatable"); + std::fs::write(wrapper_dir.join("wrapper.h"), b"x").expect("header must be writable"); + std::fs::write(wrapper_dir.join("wrapper_reasoning.cpp"), b"x") + .expect("source must be writable"); + + wrapper_dir + } + + fn llama_src_tree(scratch: &ScratchDir) -> PathBuf { + let llama_src = scratch.path().join("llama.cpp"); + std::fs::create_dir_all(llama_src.join("src")).expect("src must be creatable"); + std::fs::create_dir_all(llama_src.join("ggml/src")).expect("ggml/src must be creatable"); + std::fs::create_dir_all(llama_src.join("common")).expect("common must be creatable"); + std::fs::create_dir_all(llama_src.join(".git")).expect(".git must be creatable"); + std::fs::write(llama_src.join("CMakeLists.txt"), b"x").expect("cmake must be writable"); + std::fs::write(llama_src.join("src/llama.cpp"), b"x").expect("source must be writable"); + std::fs::write(llama_src.join(".git/config"), b"x").expect("git file must be writable"); + + llama_src + } + + #[test] + fn dot_prefixed_entries_are_hidden() { + let scratch = ScratchDir::new("hidden-yes"); + std::fs::create_dir_all(scratch.path().join(".git")).expect("dir must be creatable"); + + assert!(is_hidden(&entry_for(scratch.path()))); + } + + #[test] + fn ordinary_entries_are_not_hidden() { + let scratch = ScratchDir::new("hidden-no"); + std::fs::write(scratch.path().join("visible.txt"), b"x").expect("file must be writable"); + + assert!(!is_hidden(&entry_for(scratch.path()))); + } + + #[test] + fn cmake_prefixed_entries_are_recognised() { + let scratch = ScratchDir::new("cmake-yes"); + std::fs::write(scratch.path().join("CMakeLists.txt"), b"x").expect("file must be writable"); + + assert!(is_cmake_file(&entry_for(scratch.path()))); + } + + #[test] + fn other_entries_are_not_cmake_files() { + let scratch = ScratchDir::new("cmake-no"); + std::fs::write(scratch.path().join("readme.md"), b"x").expect("file must be writable"); + + assert!(!is_cmake_file(&entry_for(scratch.path()))); + } + + #[test] + fn registering_triggers_walks_wrappers_and_sources() { + let scratch = ScratchDir::new("triggers-ok"); + let wrapper_dir = wrapper_dir_with_sources(&scratch); + let llama_src = llama_src_tree(&scratch); + + register_rebuild_triggers(&wrapper_dir, &llama_src); + } + + #[test] + #[should_panic(expected = "wrapper rebuild tracking failed")] + fn a_wrapper_directory_without_sources_panics() { + let scratch = ScratchDir::new("triggers-missing"); + let wrapper_dir = scratch.path().join("empty"); + std::fs::create_dir_all(&wrapper_dir).expect("wrapper dir must be creatable"); + + register_rebuild_triggers(&wrapper_dir, &llama_src_tree(&scratch)); + } + + #[cfg(unix)] + #[test] + fn unreadable_directories_are_skipped_during_the_walk() { + use std::os::unix::fs::PermissionsExt; + + let scratch = ScratchDir::new("triggers-unreadable"); + let wrapper_dir = wrapper_dir_with_sources(&scratch); + let llama_src = llama_src_tree(&scratch); + let locked = llama_src.join("src/locked"); + std::fs::create_dir_all(&locked).expect("nested dir must be creatable"); + std::fs::write(locked.join("inner.c"), b"x").expect("file must be writable"); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)) + .expect("permissions must be settable"); + + register_rebuild_triggers(&wrapper_dir, &llama_src); + + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)) + .expect("permissions must be restorable for cleanup"); + } +} diff --git a/llama-cpp-bindings-build/src/scratch_dir.rs b/llama-cpp-bindings-build/src/scratch_dir.rs new file mode 100644 index 000000000..5a1b69bbf --- /dev/null +++ b/llama-cpp-bindings-build/src/scratch_dir.rs @@ -0,0 +1,39 @@ +use std::env; +use std::path::Path; +use std::path::PathBuf; +use std::sync::atomic::AtomicU32; +use std::sync::atomic::Ordering; + +static NEXT_SCRATCH_ID: AtomicU32 = AtomicU32::new(0); + +/// A real directory under the system temp dir, removed when the guard drops. +/// +/// Build-script logic is filesystem logic, so its tests operate on real +/// directories rather than stand-ins. +pub struct ScratchDir { + path: PathBuf, +} + +impl ScratchDir { + pub fn new(label: &str) -> Self { + let unique = NEXT_SCRATCH_ID.fetch_add(1, Ordering::Relaxed); + let path = env::temp_dir().join(format!( + "llama-cpp-bindings-build-{label}-{}-{unique}", + std::process::id() + )); + + std::fs::create_dir_all(&path).expect("scratch directory must be creatable"); + + Self { path } + } + + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for ScratchDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} diff --git a/llama-cpp-bindings-build/src/shared_libs.rs b/llama-cpp-bindings-build/src/shared_libs.rs index 682882772..c0de25bf8 100644 --- a/llama-cpp-bindings-build/src/shared_libs.rs +++ b/llama-cpp-bindings-build/src/shared_libs.rs @@ -7,9 +7,7 @@ pub fn copy_shared_libraries(cmake_dir: &Path, target_dir: &Path) { let assets = extract_lib_assets(cmake_dir); for asset in &assets { - let Some(filename) = asset.file_name().and_then(|name| name.to_str()) else { - continue; - }; + let filename = asset.file_name().unwrap_or_else(|| asset.as_os_str()); hard_link_if_missing(asset, &target_dir.join(filename)); @@ -43,3 +41,139 @@ fn hard_link_if_missing(source: &Path, destination: &Path) { ); } } + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::path::PathBuf; + + use serial_test::serial; + + use crate::scratch_dir::ScratchDir; + + use super::copy_shared_libraries; + use super::hard_link_if_missing; + + fn shared_lib_name(stem: &str) -> String { + crate::host_platform::HostPlatform::current() + .shared_library_pattern() + .replace('*', stem) + } + + fn cmake_dir_with_one_library(scratch: &ScratchDir) -> PathBuf { + let cmake_dir = scratch.path().join("cmake"); + let assets_dir = + cmake_dir.join(crate::host_platform::HostPlatform::current().shared_library_dir()); + std::fs::create_dir_all(&assets_dir).expect("assets directory must be creatable"); + std::fs::write(assets_dir.join(shared_lib_name("libggml")), b"payload") + .expect("shared library must be writable"); + + cmake_dir + } + + fn target_dir_with_deps(scratch: &ScratchDir) -> PathBuf { + let target_dir = scratch.path().join("target"); + std::fs::create_dir_all(target_dir.join("deps")).expect("deps must be creatable"); + + target_dir + } + + #[test] + fn libraries_are_linked_into_the_target_root_and_deps() { + let scratch = ScratchDir::new("shared-basic"); + let cmake_dir = cmake_dir_with_one_library(&scratch); + let target_dir = target_dir_with_deps(&scratch); + + copy_shared_libraries(&cmake_dir, &target_dir); + + let name = shared_lib_name("libggml"); + assert!(target_dir.join(&name).exists(), "root copy must exist"); + assert!( + target_dir.join("deps").join(&name).exists(), + "deps copy must exist" + ); + assert!( + !target_dir.join("examples").join(&name).exists(), + "examples must be skipped when the directory is absent" + ); + } + + #[test] + fn an_existing_examples_directory_also_receives_the_library() { + let scratch = ScratchDir::new("shared-examples"); + let cmake_dir = cmake_dir_with_one_library(&scratch); + let target_dir = target_dir_with_deps(&scratch); + std::fs::create_dir_all(target_dir.join("examples")).expect("examples must be creatable"); + + copy_shared_libraries(&cmake_dir, &target_dir); + + assert!( + target_dir + .join("examples") + .join(shared_lib_name("libggml")) + .exists() + ); + } + + #[test] + fn an_existing_destination_is_left_untouched() { + let scratch = ScratchDir::new("shared-existing"); + let source = scratch.path().join("source.bin"); + let destination = scratch.path().join("destination.bin"); + std::fs::write(&source, b"fresh").expect("source must be writable"); + std::fs::write(&destination, b"original").expect("destination must be writable"); + + hard_link_if_missing(&source, &destination); + + assert_eq!( + std::fs::read(&destination).expect("destination must be readable"), + b"original", + "an existing destination must not be replaced" + ); + } + + #[test] + fn a_failing_link_is_reported_without_panicking() { + let scratch = ScratchDir::new("shared-failure"); + let source = scratch.path().join("source.bin"); + std::fs::write(&source, b"payload").expect("source must be writable"); + let destination = scratch.path().join("absent-parent").join("destination.bin"); + + hard_link_if_missing(&source, &destination); + + assert!( + !destination.exists(), + "linking into a missing parent cannot succeed" + ); + } + + #[test] + fn a_cmake_directory_without_libraries_links_nothing() { + let scratch = ScratchDir::new("shared-empty"); + let target_dir = target_dir_with_deps(&scratch); + + copy_shared_libraries(Path::new(scratch.path()), &target_dir); + + assert!( + std::fs::read_dir(target_dir.join("deps")) + .expect("deps must be readable") + .next() + .is_none() + ); + } + + #[test] + #[serial] + fn debug_logging_reports_each_link() { + let scratch = ScratchDir::new("shared-debug"); + let source = scratch.path().join("source.bin"); + std::fs::write(&source, b"payload").expect("source must be writable"); + unsafe { std::env::set_var("BUILD_DEBUG", "1") }; + + hard_link_if_missing(&source, &scratch.path().join("linked.bin")); + + unsafe { std::env::remove_var("BUILD_DEBUG") }; + + assert!(scratch.path().join("linked.bin").exists()); + } +} diff --git a/llama-cpp-bindings-build/src/stable_cmake_build_dir.rs b/llama-cpp-bindings-build/src/stable_cmake_build_dir.rs index 87877e803..71016bf8c 100644 --- a/llama-cpp-bindings-build/src/stable_cmake_build_dir.rs +++ b/llama-cpp-bindings-build/src/stable_cmake_build_dir.rs @@ -49,3 +49,84 @@ pub fn stable_cmake_build_dir( path } + +#[cfg(test)] +mod tests { + use serial_test::serial; + + use crate::scratch_dir::ScratchDir; + + use super::stable_cmake_build_dir; + + const OVERRIDE_VAR: &str = "LLAMA_CMAKE_BUILD_DIR_OVERRIDE"; + + fn build_dir_for( + scratch: &ScratchDir, + target_triple: &str, + profile: &str, + ) -> std::path::PathBuf { + stable_cmake_build_dir(scratch.path(), target_triple, profile, false, false) + } + + #[test] + #[serial] + fn the_same_inputs_always_yield_the_same_directory() { + unsafe { std::env::remove_var(OVERRIDE_VAR) }; + let scratch = ScratchDir::new("cmakedir-stable"); + + let first = build_dir_for(&scratch, "aarch64-apple-darwin", "Release"); + let second = build_dir_for(&scratch, "aarch64-apple-darwin", "Release"); + + assert_eq!(first, second, "the directory must be stable across calls"); + assert!(first.is_dir(), "the directory must be created"); + } + + #[test] + #[serial] + fn each_configuration_input_changes_the_directory() { + unsafe { std::env::remove_var(OVERRIDE_VAR) }; + let scratch = ScratchDir::new("cmakedir-varies"); + let baseline = build_dir_for(&scratch, "aarch64-apple-darwin", "Release"); + + let other_triple = build_dir_for(&scratch, "x86_64-unknown-linux-gnu", "Release"); + let other_profile = build_dir_for(&scratch, "aarch64-apple-darwin", "Debug"); + let static_crt = stable_cmake_build_dir( + scratch.path(), + "aarch64-apple-darwin", + "Release", + true, + false, + ); + let shared_libs = stable_cmake_build_dir( + scratch.path(), + "aarch64-apple-darwin", + "Release", + false, + true, + ); + + for (label, candidate) in [ + ("target triple", other_triple), + ("profile", other_profile), + ("static crt", static_crt), + ("shared libs", shared_libs), + ] { + assert_ne!(baseline, candidate, "{label} must change the build dir"); + } + } + + #[test] + #[serial] + fn the_override_environment_variable_wins_and_is_created() { + let scratch = ScratchDir::new("cmakedir-override"); + let override_path = scratch.path().join("explicit-build-dir"); + unsafe { std::env::set_var(OVERRIDE_VAR, &override_path) }; + + let resolved = build_dir_for(&scratch, "aarch64-apple-darwin", "Release"); + + unsafe { std::env::remove_var(OVERRIDE_VAR) }; + + assert_eq!(resolved, override_path); + assert!(resolved.is_dir(), "the override directory must be created"); + } +} diff --git a/llama-cpp-bindings-build/src/target_os.rs b/llama-cpp-bindings-build/src/target_os.rs index 9874bf25f..2483f5135 100644 --- a/llama-cpp-bindings-build/src/target_os.rs +++ b/llama-cpp-bindings-build/src/target_os.rs @@ -49,3 +49,81 @@ impl TargetOs { matches!(self, TargetOs::Windows(WindowsVariant::Msvc)) } } + +#[cfg(test)] +mod tests { + use super::AppleVariant; + use super::TargetOs; + use super::WindowsVariant; + + #[test] + fn msvc_triple_is_distinguished_from_other_windows_targets() { + assert!(matches!( + TargetOs::from_target_triple("x86_64-pc-windows-msvc"), + Ok(TargetOs::Windows(WindowsVariant::Msvc)) + )); + assert!(matches!( + TargetOs::from_target_triple("x86_64-pc-windows-gnu"), + Ok(TargetOs::Windows(WindowsVariant::Other)) + )); + } + + #[test] + fn macos_triple_is_distinguished_from_other_apple_targets() { + assert!(matches!( + TargetOs::from_target_triple("aarch64-apple-darwin"), + Ok(TargetOs::Apple(AppleVariant::MacOS)) + )); + assert!(matches!( + TargetOs::from_target_triple("aarch64-apple-ios"), + Ok(TargetOs::Apple(AppleVariant::Other)) + )); + } + + #[test] + fn android_is_matched_before_linux_despite_containing_linux() { + assert!(matches!( + TargetOs::from_target_triple("aarch64-linux-android"), + Ok(TargetOs::Android) + )); + } + + #[test] + fn plain_linux_triple_is_linux() { + assert!(matches!( + TargetOs::from_target_triple("x86_64-unknown-linux-gnu"), + Ok(TargetOs::Linux) + )); + } + + #[test] + fn an_unsupported_triple_reports_the_triple() { + let error = TargetOs::from_target_triple("sparc-unknown-none") + .expect_err("unsupported triple must fail"); + + assert!(error.contains("sparc-unknown-none"), "got: {error}"); + } + + #[test] + fn is_android_and_is_msvc_only_match_their_own_variant() { + let android = TargetOs::from_target_triple("aarch64-linux-android").expect("android"); + let msvc = TargetOs::from_target_triple("x86_64-pc-windows-msvc").expect("msvc"); + let linux = TargetOs::from_target_triple("x86_64-unknown-linux-gnu").expect("linux"); + + assert!(android.is_android()); + assert!(!android.is_msvc()); + assert!(msvc.is_msvc()); + assert!(!msvc.is_android()); + assert!(!linux.is_android()); + assert!(!linux.is_msvc()); + } + + #[test] + fn variants_are_debug_printable_for_build_diagnostics() { + let msvc = TargetOs::from_target_triple("x86_64-pc-windows-msvc").expect("msvc"); + let macos = TargetOs::from_target_triple("aarch64-apple-darwin").expect("macos"); + + assert!(format!("{msvc:?}").contains("Msvc")); + assert!(format!("{macos:?}").contains("MacOS")); + } +} diff --git a/llama-cpp-bindings-sys/Cargo.toml b/llama-cpp-bindings-sys/Cargo.toml index 7a8342938..f696562e4 100644 --- a/llama-cpp-bindings-sys/Cargo.toml +++ b/llama-cpp-bindings-sys/Cargo.toml @@ -57,7 +57,6 @@ include = [ "/llama.cpp/cmake", "/llama.cpp/ggml/cmake", - "/llama.cpp/common/cmake", ] [dependencies] diff --git a/llama-cpp-bindings-sys/llama.cpp b/llama-cpp-bindings-sys/llama.cpp index 571d0d540..6e62ba538 160000 --- a/llama-cpp-bindings-sys/llama.cpp +++ b/llama-cpp-bindings-sys/llama.cpp @@ -1 +1 @@ -Subproject commit 571d0d540df04f25298d0e159e520d9fc62ed121 +Subproject commit 6e62ba538478202094edc6c100c782719e310aa3 diff --git a/llama-cpp-bindings-sys/wrapper_common.cpp b/llama-cpp-bindings-sys/wrapper_common.cpp index cc20c9af5..1016403ce 100644 --- a/llama-cpp-bindings-sys/wrapper_common.cpp +++ b/llama-cpp-bindings-sys/wrapper_common.cpp @@ -77,6 +77,17 @@ extern "C" void llama_rs_string_free(char * ptr) { const std::unique_ptr reclaimed(ptr); } +extern "C" void llama_rs_string_array_free(char ** strings, size_t count) { + if (strings == nullptr) { + return; + } + const gsl::span entries(strings, count); + for (char * entry : entries) { + const std::unique_ptr reclaimed(entry); + } + const std::unique_ptr reclaimed_array(strings); +} + extern "C" auto llama_rs_sampler_init_grammar( const struct llama_vocab * vocab, const char * grammar_str, diff --git a/llama-cpp-bindings-sys/wrapper_common.h b/llama-cpp-bindings-sys/wrapper_common.h index 7896e200b..e02af2500 100644 --- a/llama-cpp-bindings-sys/wrapper_common.h +++ b/llama-cpp-bindings-sys/wrapper_common.h @@ -121,6 +121,8 @@ llama_rs_sampler_sample_status llama_rs_sampler_sample( void llama_rs_string_free(char * ptr); +void llama_rs_string_array_free(char ** strings, size_t count); + llama_pos llama_rs_memory_seq_pos_max( const struct llama_context * ctx, llama_seq_id seq_id); diff --git a/llama-cpp-bindings-sys/wrapper_reasoning.cpp b/llama-cpp-bindings-sys/wrapper_reasoning.cpp index 5fcf90949..68eed95db 100644 --- a/llama-cpp-bindings-sys/wrapper_reasoning.cpp +++ b/llama-cpp-bindings-sys/wrapper_reasoning.cpp @@ -7,11 +7,15 @@ #include #include "wrapper_utils.h" +#include +#include #include +#include #include #include #include #include +#include namespace { @@ -32,7 +36,7 @@ auto find_reasoning_markers( const common_chat_template & tmpl, const char * tmpl_src, std::string * out_start, - std::string * out_end) -> bool { + std::vector * out_ends) -> bool { autoparser::generation_params probe_params; probe_params.add_generation_prompt = true; probe_params.enable_thinking = true; @@ -47,9 +51,9 @@ auto find_reasoning_markers( if (auto specialized = common_chat_try_specialized_template(tmpl, tmpl_src_str, probe_params)) { if (specialized->supports_thinking && !specialized->thinking_start_tag.empty() - && !specialized->thinking_end_tag.empty()) { + && !specialized->thinking_end_tags.empty()) { *out_start = std::move(specialized->thinking_start_tag); - *out_end = std::move(specialized->thinking_end_tag); + *out_ends = std::move(specialized->thinking_end_tags); return true; } } @@ -60,7 +64,8 @@ auto find_reasoning_markers( && !parser.reasoning.start.empty() && !parser.reasoning.end.empty()) { *out_start = std::move(parser.reasoning.start); - *out_end = std::move(parser.reasoning.end); + out_ends->clear(); + out_ends->push_back(std::move(parser.reasoning.end)); return true; } @@ -72,13 +77,17 @@ auto find_reasoning_markers( extern "C" auto llama_rs_detect_reasoning_markers( const struct llama_model * model, char ** out_open, - char ** out_close, + char *** out_closes, + size_t * out_closes_count, char ** out_error) -> llama_rs_detect_reasoning_markers_status { if (out_open != nullptr) { *out_open = nullptr; } - if (out_close != nullptr) { - *out_close = nullptr; + if (out_closes != nullptr) { + *out_closes = nullptr; + } + if (out_closes_count != nullptr) { + *out_closes_count = 0; } if (out_error != nullptr) { *out_error = nullptr; @@ -89,8 +98,11 @@ extern "C" auto llama_rs_detect_reasoning_markers( if (out_open == nullptr) { return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_OPEN_ARG; } - if (out_close == nullptr) { - return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_CLOSE_ARG; + if (out_closes == nullptr) { + return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_CLOSES_ARG; + } + if (out_closes_count == nullptr) { + return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_CLOSES_COUNT_ARG; } if (out_error == nullptr) { return LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG; @@ -113,20 +125,40 @@ extern "C" auto llama_rs_detect_reasoning_markers( common_chat_template const tmpl(tmpl_src, bos_token, eos_token); std::string detected_start; - std::string detected_end; - if (!find_reasoning_markers(tmpl, tmpl_src, &detected_start, &detected_end)) { + std::vector detected_ends; + if (!find_reasoning_markers(tmpl, tmpl_src, &detected_start, &detected_ends)) { return LLAMA_RS_DETECT_REASONING_MARKERS_OK; } std::unique_ptr open_dup(llama_rs_dup_string(detected_start)); - std::unique_ptr close_dup(llama_rs_dup_string(detected_end)); + if (open_dup == nullptr) { + return LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED; + } + + std::vector> close_dups; + close_dups.reserve(detected_ends.size()); + for (const std::string & detected_end : detected_ends) { + std::unique_ptr close_dup(llama_rs_dup_string(detected_end)); + if (close_dup == nullptr) { + return LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED; + } + close_dups.push_back(std::move(close_dup)); + } - if ((open_dup == nullptr) || (close_dup == nullptr)) { + std::unique_ptr closes_array(new (std::nothrow) char *[close_dups.size()]); + if (closes_array == nullptr) { return LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED; } + const gsl::span closes_view(closes_array.get(), close_dups.size()); + std::transform( + close_dups.begin(), + close_dups.end(), + closes_view.begin(), + [](auto & close_dup) -> char * { return close_dup.release(); }); *out_open = open_dup.release(); - *out_close = close_dup.release(); + *out_closes_count = close_dups.size(); + *out_closes = closes_array.release(); return LLAMA_RS_DETECT_REASONING_MARKERS_OK; } catch (const std::bad_alloc &) { diff --git a/llama-cpp-bindings-sys/wrapper_reasoning.h b/llama-cpp-bindings-sys/wrapper_reasoning.h index acf383964..de9b57b1e 100644 --- a/llama-cpp-bindings-sys/wrapper_reasoning.h +++ b/llama-cpp-bindings-sys/wrapper_reasoning.h @@ -11,7 +11,8 @@ typedef enum llama_rs_detect_reasoning_markers_status { LLAMA_RS_DETECT_REASONING_MARKERS_OK = 0, LLAMA_RS_DETECT_REASONING_MARKERS_NULL_MODEL_ARG, LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_OPEN_ARG, - LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_CLOSE_ARG, + LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_CLOSES_ARG, + LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_CLOSES_COUNT_ARG, LLAMA_RS_DETECT_REASONING_MARKERS_NULL_OUT_ERROR_ARG, LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED, LLAMA_RS_DETECT_REASONING_MARKERS_VENDORED_THREW_CXX_EXCEPTION, @@ -20,7 +21,8 @@ typedef enum llama_rs_detect_reasoning_markers_status { llama_rs_detect_reasoning_markers_status llama_rs_detect_reasoning_markers( const struct llama_model * model, char ** out_open, - char ** out_close, + char *** out_closes, + size_t * out_closes_count, char ** out_error); typedef enum llama_rs_render_chat_template_status { diff --git a/llama-cpp-bindings-tests/tests/backend_initialization.rs b/llama-cpp-bindings-tests/tests/backend_initialization.rs index 36f82b103..6bb3bfa31 100644 --- a/llama-cpp-bindings-tests/tests/backend_initialization.rs +++ b/llama-cpp-bindings-tests/tests/backend_initialization.rs @@ -5,8 +5,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, diff --git a/llama-cpp-bindings-tests/tests/chat_template_and_message_parsing.rs b/llama-cpp-bindings-tests/tests/chat_template_and_message_parsing.rs index d21d00c43..7f8d091f9 100644 --- a/llama-cpp-bindings-tests/tests/chat_template_and_message_parsing.rs +++ b/llama-cpp-bindings-tests/tests/chat_template_and_message_parsing.rs @@ -10,8 +10,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -19,8 +18,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -28,8 +26,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -37,8 +34,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -52,8 +48,7 @@ fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -61,8 +56,7 @@ fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -70,8 +64,7 @@ fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -79,8 +72,7 @@ fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -88,8 +80,7 @@ fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -100,8 +91,7 @@ fn chat_template_returns_non_empty(fixture: &LlamaFixture<'_>) -> Result<()> { "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf" ), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -123,8 +113,7 @@ fn apply_chat_template_produces_prompt(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 64, n_batch = 64, n_ubatch = 64, @@ -142,8 +131,7 @@ fn build_user_prompt_surfaces_message_construction_error(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -151,8 +139,7 @@ fn build_user_prompt_surfaces_message_construction_error(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -160,8 +147,7 @@ fn build_user_prompt_surfaces_message_construction_error(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -169,8 +155,7 @@ fn build_user_prompt_surfaces_message_construction_error(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -192,8 +177,7 @@ fn apply_chat_template_renders_long_messages(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -201,8 +185,7 @@ fn apply_chat_template_renders_long_messages(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -210,8 +193,7 @@ fn apply_chat_template_renders_long_messages(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -219,8 +201,7 @@ fn apply_chat_template_renders_long_messages(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -236,8 +217,7 @@ fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -245,8 +225,7 @@ fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -254,8 +233,7 @@ fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -263,8 +241,7 @@ fn chat_template_with_nonexistent_name_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -287,8 +264,7 @@ fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -296,8 +272,7 @@ fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -305,8 +280,7 @@ fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -314,8 +288,7 @@ fn parses_pure_content_response(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -340,8 +313,7 @@ fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -349,8 +321,7 @@ fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -358,8 +329,7 @@ fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -367,8 +337,7 @@ fn parses_reasoning_section_into_reasoning_content(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -387,8 +356,7 @@ fn parses_empty_input_yields_empty_message(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -396,8 +364,7 @@ fn parses_empty_input_yields_empty_message(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -405,8 +372,7 @@ fn parses_empty_input_yields_empty_message(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -414,8 +380,7 @@ fn parses_empty_input_yields_empty_message(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -439,8 +404,7 @@ fn parses_malformed_tools_json_returns_tools_json_invalid_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -448,8 +412,7 @@ fn parses_malformed_tools_json_returns_tools_json_invalid_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -457,8 +420,7 @@ fn parses_malformed_tools_json_returns_tools_json_invalid_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -466,8 +428,7 @@ fn parses_malformed_tools_json_returns_tools_json_invalid_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -489,8 +450,7 @@ fn parses_non_array_tools_json_returns_tools_json_not_array_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -498,8 +458,7 @@ fn parses_non_array_tools_json_returns_tools_json_not_array_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -507,8 +466,7 @@ fn parses_non_array_tools_json_returns_tools_json_not_array_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -516,8 +474,7 @@ fn parses_non_array_tools_json_returns_tools_json_not_array_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -541,8 +498,7 @@ fn parses_with_tools_null_byte_returns_tools_json_invalid_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -550,8 +506,7 @@ fn parses_with_tools_null_byte_returns_tools_json_invalid_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -559,8 +514,7 @@ fn parses_with_tools_null_byte_returns_tools_json_invalid_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -568,8 +522,7 @@ fn parses_with_tools_null_byte_returns_tools_json_invalid_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, diff --git a/llama-cpp-bindings-tests/tests/embedding_and_encoder.rs b/llama-cpp-bindings-tests/tests/embedding_and_encoder.rs index f681f5b59..141aa5a88 100644 --- a/llama-cpp-bindings-tests/tests/embedding_and_encoder.rs +++ b/llama-cpp-bindings-tests/tests/embedding_and_encoder.rs @@ -32,8 +32,7 @@ fn cosine_similarity(vec_a: &[f32], vec_b: &[f32]) -> f32 { #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -114,8 +113,7 @@ fn embedding_generation_produces_vectors(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -239,8 +237,7 @@ fn reranking_produces_scores(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -266,8 +263,7 @@ fn decode_with_embeddings_enabled(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -294,8 +290,7 @@ fn embeddings_seq_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -359,8 +354,7 @@ fn multi_sequence_embeddings_returns_one_embedding_per_sequence( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -439,8 +433,7 @@ fn embeddings_returns_distinct_values_when_reused_batch_has_extra_capacity( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -468,8 +461,7 @@ fn embeddings_ith_returns_valid_embeddings(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -494,8 +486,7 @@ fn embeddings_ith_returns_null_embedding_error_for_non_embedding_token( #[llama_test( model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -521,8 +512,7 @@ fn encode_succeeds_with_encoder_model(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -542,8 +532,7 @@ fn kv_cache_seq_add_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -564,8 +553,7 @@ fn kv_cache_seq_div_succeeds_on_embedding_model(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128 @@ -579,8 +567,7 @@ fn embedding_model_tool_call_markers_call_does_not_panic(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128 @@ -596,8 +583,7 @@ fn embedding_model_streaming_markers_returns_ok_for_a_model_without_tool_calls( #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128 @@ -616,3 +602,25 @@ fn approximate_tok_env_falls_back_to_eos_when_eot_unavailable( Ok(()) } + +#[llama_test( + model_source = HuggingFace("Xiaojian9992024/t5-small-GGUF", "t5-small.bf16.gguf"), + n_gpu_layers = 999, + load_mode = Auto, + n_ctx = 512, + n_batch = 2048, + n_ubatch = 512, + embeddings = true, +)] +fn tool_call_markers_are_absent_for_a_model_without_a_chat_template( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let markers = fixture.model.tool_call_markers()?; + + assert!( + markers.is_none(), + "an encoder model carries no chat template, so it advertises no tool-call markers; got: {markers:?}" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/kv_cache_and_session.rs b/llama-cpp-bindings-tests/tests/kv_cache_and_session.rs index e6ad1e511..6dda790ca 100644 --- a/llama-cpp-bindings-tests/tests/kv_cache_and_session.rs +++ b/llama-cpp-bindings-tests/tests/kv_cache_and_session.rs @@ -21,8 +21,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 256, n_batch = 128, n_ubatch = 64, @@ -30,8 +29,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 256, n_batch = 128, n_ubatch = 64, @@ -39,8 +37,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 256, n_batch = 128, n_ubatch = 64, @@ -48,8 +45,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 256, n_batch = 128, n_ubatch = 64, @@ -68,8 +64,7 @@ fn new_context_returns_valid_context(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4294967295, n_batch = 128, n_ubatch = 64, @@ -77,8 +72,7 @@ fn new_context_returns_valid_context(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4294967295, n_batch = 128, n_ubatch = 64, @@ -86,8 +80,7 @@ fn new_context_returns_valid_context(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4294967295, n_batch = 128, n_ubatch = 64, @@ -95,8 +88,7 @@ fn new_context_returns_valid_context(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4294967295, n_batch = 128, n_ubatch = 64, @@ -111,8 +103,7 @@ fn new_context_with_huge_ctx_returns_null_error(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -134,8 +125,7 @@ fn context_creation_and_properties(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -162,8 +152,7 @@ fn decode_and_get_logits(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -185,8 +174,7 @@ fn timings_work(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -212,8 +200,7 @@ fn token_data_array_has_entries_after_decode(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -240,8 +227,7 @@ fn get_logits_ith_returns_valid_slice(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -271,8 +257,7 @@ fn token_data_array_ith_returns_valid_data(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -294,8 +279,7 @@ fn embeddings_ith_returns_error_when_embeddings_disabled(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -319,8 +303,7 @@ fn embeddings_seq_ith_returns_error_when_embeddings_disabled( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -346,8 +329,7 @@ fn candidates_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -368,8 +350,7 @@ fn debug_format_contains_struct_name(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -396,8 +377,7 @@ fn candidates_ith_returns_n_vocab_entries(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -422,8 +402,7 @@ fn lora_adapter_remove_succeeds_with_no_adapters(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -448,8 +427,7 @@ fn encode_on_non_encoder_model_returns_error(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -476,8 +454,7 @@ fn lora_adapter_set_with_dangling_pointer_succeeds_or_errors( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -506,8 +483,7 @@ fn embeddings_seq_ith_returns_null_embedding_error_for_invalid_seq( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -530,8 +506,7 @@ fn decode_empty_batch_returns_error(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -559,8 +534,7 @@ fn set_abort_flag_aborts_decode(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -588,8 +562,7 @@ fn set_abort_flag_false_allows_decode(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -618,8 +591,7 @@ fn clear_abort_callback_allows_decode_with_flag_true(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -639,8 +611,7 @@ fn synchronize_completes_without_panic(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -660,8 +631,7 @@ fn detach_threadpool_completes_without_panic(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -685,8 +655,7 @@ fn get_logits_ith_returns_token_not_initialized_for_unknown_index( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 64, n_batch = 2048, n_ubatch = 512, @@ -715,8 +684,7 @@ fn get_logits_ith_returns_token_index_exceeds_context_for_huge_index( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -724,8 +692,7 @@ fn get_logits_ith_returns_token_index_exceeds_context_for_huge_index( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -733,8 +700,7 @@ fn get_logits_ith_returns_token_index_exceeds_context_for_huge_index( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -742,8 +708,7 @@ fn get_logits_ith_returns_token_index_exceeds_context_for_huge_index( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -762,8 +727,7 @@ fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -771,8 +735,7 @@ fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -780,8 +743,7 @@ fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -789,8 +751,7 @@ fn clear_kv_cache_resets_positions(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -808,8 +769,7 @@ fn kv_cache_seq_pos_max_is_non_negative_after_decode(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 256, n_batch = 256, n_ubatch = 64, @@ -848,8 +808,7 @@ fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -857,8 +816,7 @@ fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -866,8 +824,7 @@ fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -875,8 +832,7 @@ fn prime_kv_cache_surfaces_each_underlying_error(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -895,8 +851,7 @@ fn clear_kv_cache_seq_with_range(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -904,8 +859,7 @@ fn clear_kv_cache_seq_with_range(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -913,8 +867,7 @@ fn clear_kv_cache_seq_with_range(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -922,8 +875,7 @@ fn clear_kv_cache_seq_with_range(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -942,8 +894,7 @@ fn copy_kv_cache_seq_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -951,8 +902,7 @@ fn copy_kv_cache_seq_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -960,8 +910,7 @@ fn copy_kv_cache_seq_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -969,8 +918,7 @@ fn copy_kv_cache_seq_succeeds(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -989,8 +937,7 @@ fn copy_cache_executes_without_crash(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -998,8 +945,7 @@ fn copy_cache_executes_without_crash(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1022,8 +968,7 @@ fn kv_cache_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1031,8 +976,7 @@ fn kv_cache_seq_add_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1056,8 +1000,7 @@ fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1065,8 +1008,7 @@ fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1074,8 +1016,7 @@ fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1083,8 +1024,7 @@ fn kv_cache_seq_div_returns_error_for_mrope_model(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1104,8 +1044,7 @@ fn kv_cache_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1113,8 +1052,7 @@ fn kv_cache_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1122,8 +1060,7 @@ fn kv_cache_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1131,8 +1068,7 @@ fn kv_cache_seq_keep_retains_specified_sequence(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1152,8 +1088,7 @@ fn copy_kv_cache_seq_with_explicit_range(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1161,8 +1096,7 @@ fn copy_kv_cache_seq_with_explicit_range(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1170,8 +1104,7 @@ fn copy_kv_cache_seq_with_explicit_range(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1179,8 +1112,7 @@ fn copy_kv_cache_seq_with_explicit_range(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1200,8 +1132,7 @@ fn kv_cache_seq_pos_max_returns_negative_one_for_unused_seq( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1209,8 +1140,7 @@ fn kv_cache_seq_pos_max_returns_negative_one_for_unused_seq( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1218,8 +1148,7 @@ fn kv_cache_seq_pos_max_returns_negative_one_for_unused_seq( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1227,8 +1156,7 @@ fn kv_cache_seq_pos_max_returns_negative_one_for_unused_seq( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1249,8 +1177,7 @@ fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1258,8 +1185,7 @@ fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1267,8 +1193,7 @@ fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1276,8 +1201,7 @@ fn copy_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1298,8 +1222,7 @@ fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1307,8 +1230,7 @@ fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1316,8 +1238,7 @@ fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1325,8 +1246,7 @@ fn copy_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1347,8 +1267,7 @@ fn clear_kv_cache_seq_rejects_src_exceeding_i32_max(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1356,8 +1275,7 @@ fn clear_kv_cache_seq_rejects_src_exceeding_i32_max(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1365,8 +1283,7 @@ fn clear_kv_cache_seq_rejects_src_exceeding_i32_max(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1374,8 +1291,7 @@ fn clear_kv_cache_seq_rejects_src_exceeding_i32_max(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1396,8 +1312,7 @@ fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1405,8 +1320,7 @@ fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1414,8 +1328,7 @@ fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1423,8 +1336,7 @@ fn clear_kv_cache_seq_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1445,8 +1357,7 @@ fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1454,8 +1365,7 @@ fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1463,8 +1373,7 @@ fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1472,8 +1381,7 @@ fn clear_kv_cache_seq_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1494,8 +1402,7 @@ fn kv_cache_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1503,8 +1410,7 @@ fn kv_cache_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1512,8 +1418,7 @@ fn kv_cache_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1521,8 +1426,7 @@ fn kv_cache_seq_add_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1543,8 +1447,7 @@ fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1552,8 +1455,7 @@ fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1561,8 +1463,7 @@ fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1570,8 +1471,7 @@ fn kv_cache_seq_add_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1593,8 +1493,7 @@ fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1602,8 +1501,7 @@ fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1611,8 +1509,7 @@ fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1620,8 +1517,7 @@ fn kv_cache_seq_div_rejects_p0_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1643,8 +1539,7 @@ fn kv_cache_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1652,8 +1547,7 @@ fn kv_cache_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1661,8 +1555,7 @@ fn kv_cache_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1670,8 +1563,7 @@ fn kv_cache_seq_div_rejects_p1_exceeding_i32_max(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1698,8 +1590,7 @@ fn save_and_load_session_file(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1707,8 +1598,7 @@ fn save_and_load_session_file(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1716,8 +1606,7 @@ fn save_and_load_session_file(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1725,8 +1614,7 @@ fn save_and_load_session_file(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1742,8 +1630,7 @@ fn get_state_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1751,8 +1638,7 @@ fn get_state_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1760,8 +1646,7 @@ fn get_state_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1769,8 +1654,7 @@ fn get_state_size_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1799,8 +1683,7 @@ fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1808,8 +1691,7 @@ fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1817,8 +1699,7 @@ fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1826,8 +1707,7 @@ fn state_seq_save_and_load_file_roundtrip(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1854,8 +1734,7 @@ fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1863,8 +1742,7 @@ fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1872,8 +1750,7 @@ fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1881,8 +1758,7 @@ fn copy_state_data_and_set_state_data_roundtrip(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1900,8 +1776,7 @@ fn state_load_file_with_nonexistent_file_returns_error(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1909,8 +1784,7 @@ fn state_load_file_with_nonexistent_file_returns_error(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1918,8 +1792,7 @@ fn state_load_file_with_nonexistent_file_returns_error(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1927,8 +1800,7 @@ fn state_load_file_with_nonexistent_file_returns_error(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1948,8 +1820,7 @@ fn state_seq_load_file_with_nonexistent_file_returns_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1957,8 +1828,7 @@ fn state_seq_load_file_with_nonexistent_file_returns_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1966,8 +1836,7 @@ fn state_seq_load_file_with_nonexistent_file_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1975,8 +1844,7 @@ fn state_seq_load_file_with_nonexistent_file_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1996,8 +1864,7 @@ fn state_save_file_to_invalid_directory_returns_failed_to_save( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2005,8 +1872,7 @@ fn state_save_file_to_invalid_directory_returns_failed_to_save( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2014,8 +1880,7 @@ fn state_save_file_to_invalid_directory_returns_failed_to_save( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2023,8 +1888,7 @@ fn state_save_file_to_invalid_directory_returns_failed_to_save( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2044,8 +1908,7 @@ fn state_seq_save_file_to_invalid_directory_returns_failed_to_save( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2053,8 +1916,7 @@ fn state_seq_save_file_to_invalid_directory_returns_failed_to_save( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2062,8 +1924,7 @@ fn state_seq_save_file_to_invalid_directory_returns_failed_to_save( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2071,8 +1932,7 @@ fn state_seq_save_file_to_invalid_directory_returns_failed_to_save( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2099,8 +1959,7 @@ fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2108,8 +1967,7 @@ fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2117,8 +1975,7 @@ fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2126,8 +1983,7 @@ fn state_load_file_with_zero_max_tokens_returns_error(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2156,8 +2012,7 @@ fn state_seq_load_file_with_zero_max_tokens_returns_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2165,8 +2020,7 @@ fn state_seq_load_file_with_zero_max_tokens_returns_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2174,8 +2028,7 @@ fn state_seq_load_file_with_zero_max_tokens_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2183,8 +2036,7 @@ fn state_seq_load_file_with_zero_max_tokens_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2216,8 +2068,7 @@ fn state_load_file_with_insufficient_max_tokens_returns_length_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2225,8 +2076,7 @@ fn state_load_file_with_insufficient_max_tokens_returns_length_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2234,8 +2084,7 @@ fn state_load_file_with_insufficient_max_tokens_returns_length_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2243,8 +2092,7 @@ fn state_load_file_with_insufficient_max_tokens_returns_length_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2277,8 +2125,7 @@ fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2286,8 +2133,7 @@ fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2295,8 +2141,7 @@ fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2304,8 +2149,7 @@ fn state_seq_load_file_with_insufficient_max_tokens_returns_length_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2328,8 +2172,7 @@ fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2337,8 +2180,7 @@ fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2346,8 +2188,7 @@ fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2355,8 +2196,7 @@ fn state_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2379,8 +2219,7 @@ fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2388,8 +2227,7 @@ fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2397,8 +2235,7 @@ fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2406,8 +2243,7 @@ fn state_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2430,8 +2266,7 @@ fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2439,8 +2274,7 @@ fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2448,8 +2282,7 @@ fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2457,8 +2290,7 @@ fn state_seq_save_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2480,8 +2312,7 @@ fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2489,8 +2320,7 @@ fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2498,8 +2328,7 @@ fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2507,8 +2336,7 @@ fn state_seq_load_file_with_non_utf8_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2527,8 +2355,7 @@ fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2536,8 +2363,7 @@ fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2545,8 +2371,7 @@ fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2554,8 +2379,7 @@ fn state_save_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2574,8 +2398,7 @@ fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2583,8 +2406,7 @@ fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2592,8 +2414,7 @@ fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2601,8 +2422,7 @@ fn state_load_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2623,8 +2443,7 @@ fn state_seq_save_file_with_null_byte_in_path_returns_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2632,8 +2451,7 @@ fn state_seq_save_file_with_null_byte_in_path_returns_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2641,8 +2459,7 @@ fn state_seq_save_file_with_null_byte_in_path_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2650,8 +2467,7 @@ fn state_seq_save_file_with_null_byte_in_path_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2672,8 +2488,7 @@ fn state_seq_load_file_with_null_byte_in_path_returns_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2681,8 +2496,7 @@ fn state_seq_load_file_with_null_byte_in_path_returns_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2690,8 +2504,7 @@ fn state_seq_load_file_with_null_byte_in_path_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2699,8 +2512,7 @@ fn state_seq_load_file_with_null_byte_in_path_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2728,8 +2540,7 @@ fn state_seq_get_size_ext_returns_size_for_decoded_sequence( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2737,8 +2548,7 @@ fn state_seq_get_size_ext_returns_size_for_decoded_sequence( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2746,8 +2556,7 @@ fn state_seq_get_size_ext_returns_size_for_decoded_sequence( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -2755,8 +2564,7 @@ fn state_seq_get_size_ext_returns_size_for_decoded_sequence( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, diff --git a/llama-cpp-bindings-tests/tests/model_loading_errors.rs b/llama-cpp-bindings-tests/tests/model_loading_errors.rs index 136ad7b42..d97e3203b 100644 --- a/llama-cpp-bindings-tests/tests/model_loading_errors.rs +++ b/llama-cpp-bindings-tests/tests/model_loading_errors.rs @@ -12,8 +12,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -33,8 +32,7 @@ fn load_model_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -60,8 +58,7 @@ fn load_model_with_invalid_file_content_returns_unloadable_or_reported( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -87,8 +84,7 @@ fn load_model_with_non_utf8_path_returns_path_to_str_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -107,8 +103,7 @@ fn lora_adapter_init_with_invalid_path_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -130,8 +125,7 @@ fn lora_adapter_init_with_invalid_gguf_returns_unloadable( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, diff --git a/llama-cpp-bindings-tests/tests/multimodal_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_audio.rs index 688fa2bd4..95592f929 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_audio.rs @@ -16,6 +16,7 @@ use llama_cpp_test_harness::llama_test; const TRANSCRIBE_SYSTEM_PROMPT: &str = "You are a speech transcription assistant. Transcribe the user's audio verbatim, \ replying with only the exact words spoken."; const TRANSCRIBE_INSTRUCTION: &str = "Transcribe the speech in this audio word for word."; +const TRANSCRIPT_PREFILL: &str = "The words spoken are:"; fn assert_audio_transcription_contains( fixture: &LlamaFixture<'_>, @@ -49,7 +50,10 @@ fn assert_audio_transcription_contains( )?, ]; let input_text = MtmdInputText { - text: model.apply_chat_template(&template, &messages, true, true)?, + text: format!( + "{}{TRANSCRIPT_PREFILL}", + model.apply_chat_template(&template, &messages, true, true)? + ), add_special: false, parse_special: true, }; @@ -136,8 +140,7 @@ fn assert_audio_transcription_contains( "Llama-3.2-1B-Instruct-Q4_K_M.gguf" ), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -149,8 +152,7 @@ fn assert_audio_transcription_contains( #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -179,8 +181,7 @@ fn audio_mmproj_reports_audio_support(fixture: &LlamaFixture<'_>) -> Result<()> "Llama-3.2-1B-Instruct-Q4_K_M.gguf" ), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -192,8 +193,7 @@ fn audio_mmproj_reports_audio_support(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -209,8 +209,7 @@ fn audio_transcribes_spoken_word(fixture: &LlamaFixture<'_>) -> Result<()> { "Llama-3.2-1B-Instruct-Q4_K_M.gguf" ), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -222,8 +221,7 @@ fn audio_transcribes_spoken_word(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, diff --git a/llama-cpp-bindings-tests/tests/multimodal_image_and_audio.rs b/llama-cpp-bindings-tests/tests/multimodal_image_and_audio.rs index 3505a96ef..4bd2d60fa 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_image_and_audio.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_image_and_audio.rs @@ -19,6 +19,7 @@ use llama_cpp_test_harness::llama_test; const MAX_GENERATED_TOKENS: i32 = 512; const DESCRIBE_INSTRUCTION: &str = "Describe the animal shown in the image, then write the exact words spoken in the audio."; +const ACCEPTED_ANIMAL_LABELS: &[&str] = &["llama", "alpaca", "sheep"]; fn build_describe_image_and_audio_prompt(model: &LlamaModel) -> Result { let marker = mtmd_default_marker()?; @@ -41,8 +42,7 @@ fn load_fixture_bitmap(mtmd_ctx: &MtmdContext, file_name: &str) -> Result) -> Result<()> { "model should generate a description from combined image and audio input" ); assert!( - description.contains("alpaca"), - "the gemma-4 vision encoder recognizes the image animals as \"alpaca\"; the assertion \ - tracks the model's actual recognition so it still proves the image reached the output; \ - got: {description:?}" + ACCEPTED_ANIMAL_LABELS + .iter() + .any(|accepted_label| description.contains(accepted_label)), + "the image shows llamas; the vision encoder's label for them varies by machine even at \ + a pinned llama.cpp commit, so any of {ACCEPTED_ANIMAL_LABELS:?} proves the image \ + reached the output; got: {description:?}" ); assert!( description.contains("fence"), diff --git a/llama-cpp-bindings-tests/tests/multimodal_vision.rs b/llama-cpp-bindings-tests/tests/multimodal_vision.rs index ab670ae2d..300a3c33b 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_vision.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_vision.rs @@ -28,8 +28,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -38,8 +37,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -65,8 +63,7 @@ fn from_buffer_creates_bitmap_from_image_bytes(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -75,8 +72,7 @@ fn from_buffer_creates_bitmap_from_image_bytes(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -96,8 +92,7 @@ fn from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -106,8 +101,7 @@ fn from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -139,8 +133,7 @@ fn copy_creates_owned_duplicate(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -149,8 +142,7 @@ fn copy_creates_owned_duplicate(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -185,8 +177,7 @@ fn encode_chunk_succeeds_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -195,8 +186,7 @@ fn encode_chunk_succeeds_for_image_chunk(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -248,8 +238,7 @@ fn tokenize_synthetic(fixture: &LlamaFixture<'_>, prompt: &str) -> Result, prompt: &str) -> Result) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -287,8 +274,7 @@ fn text_chunk_has_text_type(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -308,8 +294,7 @@ fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -318,8 +303,7 @@ fn text_chunk_returns_text_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -337,8 +321,7 @@ fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -347,8 +330,7 @@ fn chunk_n_tokens_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -366,8 +348,7 @@ fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -376,8 +357,7 @@ fn chunk_n_positions_is_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -396,8 +376,7 @@ fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -406,8 +385,7 @@ fn text_chunk_id_returns_none(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -430,8 +408,7 @@ fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -440,8 +417,7 @@ fn image_chunk_returns_none_for_text_tokens(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -464,8 +440,7 @@ fn image_chunk_id_returns_some(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -474,8 +449,7 @@ fn image_chunk_id_returns_some(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -492,8 +466,7 @@ fn init_and_supports_vision(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -502,8 +475,7 @@ fn init_and_supports_vision(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -520,8 +492,7 @@ fn init_from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -530,8 +501,7 @@ fn init_from_file_with_null_byte_in_path_returns_error(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -551,8 +521,7 @@ fn decode_use_mrope_is_true_for_qwen_vision(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -561,8 +530,7 @@ fn decode_use_mrope_is_true_for_qwen_vision(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -582,8 +550,7 @@ fn support_audio_is_false_for_vision_only_mmproj(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -592,8 +559,7 @@ fn support_audio_is_false_for_vision_only_mmproj(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -647,8 +613,7 @@ fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 64, n_batch = 64, n_ubatch = 32, @@ -657,8 +622,7 @@ fn eval_synthetic_bitmap(fixture: &LlamaFixture<'_>, width: u32, height: u32) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 64, n_batch = 64, n_ubatch = 32, @@ -692,8 +656,7 @@ fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 512, n_ubatch = 512, @@ -702,8 +665,7 @@ fn eval_chunks_returns_batch_size_exceeds_context_limit_for_huge_batch( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 512, n_ubatch = 512, @@ -751,8 +713,7 @@ fn eval_chunks_with_standard_image(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 512, n_ubatch = 512, @@ -761,8 +722,7 @@ fn eval_chunks_with_standard_image(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 512, n_ubatch = 512, @@ -785,8 +745,7 @@ fn eval_chunks_with_varied_dimensions(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 512, n_ubatch = 512, @@ -795,8 +754,7 @@ fn eval_chunks_with_varied_dimensions(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 512, n_ubatch = 512, @@ -832,8 +790,7 @@ fn eval_chunks_with_extreme_dimensions_does_not_crash(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -842,8 +799,7 @@ fn eval_chunks_with_extreme_dimensions_does_not_crash(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -870,8 +826,7 @@ fn tokenize_text_with_image(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -880,8 +835,7 @@ fn tokenize_text_with_image(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -906,8 +860,7 @@ fn tokenize_bitmap_count_mismatch_returns_error(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -916,8 +869,7 @@ fn tokenize_bitmap_count_mismatch_returns_error(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -996,8 +948,7 @@ fn drive_sampling_loop( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -1158,8 +1109,7 @@ fn build_multimodal_chunks_and_eval_into_usage( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -1182,8 +1132,7 @@ fn prompt_tokens_match_text_chunk_total(fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -1206,8 +1155,7 @@ fn input_image_tokens_match_image_chunk_total(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -1235,8 +1183,7 @@ fn input_audio_tokens_are_zero_for_image_only_input(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -1260,8 +1207,7 @@ fn completion_tokens_are_zero_after_eval_before_generation( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -1319,8 +1265,7 @@ fn text_chunk_records_prompt_tokens(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -1386,8 +1331,7 @@ fn image_chunk_records_input_image_tokens_only(fixture: &LlamaFixture<'_>) -> Re #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -1429,8 +1373,7 @@ fn text_chunk_drives_marker_state_machine_to_reasoning(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 512, n_ubatch = 512, @@ -1486,7 +1429,7 @@ fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( )?; let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -1526,8 +1469,7 @@ fn gemma4_classifier_emits_reasoning_for_multimodal_thinking_prompt( #[llama_test( model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -1623,8 +1565,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -1633,8 +1574,7 @@ fn mistral3_classifier_emits_reasoning_for_multimodal_thinking_prompt( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 4096, n_batch = 512, n_ubatch = 512, @@ -1690,7 +1630,7 @@ fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( )?; let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -1730,8 +1670,7 @@ fn qwen35_classifier_emits_reasoning_for_multimodal_thinking_prompt( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 512, n_ubatch = 512, @@ -1787,7 +1726,7 @@ fn qwen36_classifier_emits_reasoning_for_multimodal_thinking_prompt( )?; let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), diff --git a/llama-cpp-bindings-tests/tests/reasoning_markers_and_tool_calls.rs b/llama-cpp-bindings-tests/tests/reasoning_markers_and_tool_calls.rs index 8cb66d700..72a5c2163 100644 --- a/llama-cpp-bindings-tests/tests/reasoning_markers_and_tool_calls.rs +++ b/llama-cpp-bindings-tests/tests/reasoning_markers_and_tool_calls.rs @@ -20,8 +20,7 @@ use serde_json::json; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -63,7 +62,7 @@ fn deepseek_r1_8b_classifier_does_not_emit_reasoning_for_thinking_disabled_promp assert_eq!(promoted, prompt_token_count); let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -135,8 +134,7 @@ fn deepseek_r1_8b_classifier_does_not_emit_reasoning_for_thinking_disabled_promp #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -174,7 +172,7 @@ fn deepseek_r1_8b_classifier_emits_reasoning_for_thinking_enabled_prompt( assert_eq!(promoted, prompt_token_count); let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -279,8 +277,7 @@ fn assert_deepseek_r1_streams( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -341,8 +338,7 @@ fn deepseek_r1_8b_duck_types_gemma_paired_quote(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -404,8 +400,7 @@ fn deepseek_r1_8b_duck_types_glm_key_value_tags(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -466,8 +461,7 @@ fn deepseek_r1_8b_duck_types_mistral_bracketed_json(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -532,8 +526,7 @@ Paris\n\ #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -582,8 +575,7 @@ fn deepseek_r1_8b_recognizes_empty_tool_calls_when_input_is_plain_content_with_t #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -612,8 +604,7 @@ fn deepseek_r1_8b_recognizes_empty_tool_calls_when_tools_not_requested( #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -716,8 +707,7 @@ fn gemma4_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -828,8 +818,7 @@ fn gemma4_classifier_emits_reasoning_for_thinking_prompt(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -887,8 +876,7 @@ fn gemma4_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/gemma-4-E4B-it-GGUF", "gemma-4-E4B-it-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -925,8 +913,7 @@ fn gemma4_template_override_returns_full_markers(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -968,7 +955,7 @@ What is 2 + 2? assert_eq!(promoted, prompt_token_count); let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -1007,8 +994,7 @@ What is 2 + 2? #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -1049,7 +1035,7 @@ What is 2 + 2? assert_eq!(promoted, prompt_token_count); let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -1105,8 +1091,7 @@ What is 2 + 2? #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1162,8 +1147,7 @@ fn glm47_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1199,8 +1183,7 @@ fn glm47_template_override_returns_full_markers(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -1269,8 +1252,7 @@ fn mistral3_classifier_does_not_emit_reasoning_for_thinking_disabled_prompt( #[llama_test( model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -1357,8 +1339,7 @@ to the user.[/THINK]Here, provide a self-contained response.[/SYSTEM_PROMPT]\ #[llama_test( model_source = HuggingFace("unsloth/Ministral-3-14B-Reasoning-2512-GGUF", "Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1413,8 +1394,7 @@ fn mistral3_parses_tool_call_payload(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128, @@ -1479,8 +1459,7 @@ fn qwen35_chat_inference_emits_reasoning_when_template_auto_opens( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -1524,7 +1503,7 @@ What is 2 + 2?<|im_end|> assert_eq!(promoted, prompt_token_count); let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -1563,8 +1542,7 @@ What is 2 + 2?<|im_end|> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -1605,7 +1583,7 @@ What is 2 + 2?<|im_end|> assert_eq!(promoted, prompt_token_count); let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -1670,8 +1648,7 @@ fn arguments_as_json(arguments: &ToolCallArguments) -> Result<&Value> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1754,8 +1731,7 @@ get off the keyboard\n\ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1812,8 +1788,7 @@ Paris\n\ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1853,8 +1828,7 @@ fn qwen35_parses_partial_tool_call_returns_pending_state(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1913,8 +1887,7 @@ Berlin\n\ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1963,8 +1936,7 @@ fn qwen35_recognizes_empty_tool_calls_when_input_is_plain_content_with_tools_req #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128, @@ -2036,8 +2008,7 @@ fn qwen36_chat_inference_emits_reasoning_when_template_auto_opens( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -2081,7 +2052,7 @@ What is 2 + 2?<|im_end|> assert_eq!(promoted, prompt_token_count); let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -2120,8 +2091,7 @@ What is 2 + 2?<|im_end|> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 8192, n_batch = 2048, n_ubatch = 512, @@ -2162,7 +2132,7 @@ What is 2 + 2?<|im_end|> assert_eq!(promoted, prompt_token_count); let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(model.n_vocab(), 64, 1.1, 0.0, 0.0), LlamaSampler::top_k(40), LlamaSampler::top_p(0.9, 1), LlamaSampler::min_p(0.05, 1), @@ -2211,3 +2181,34 @@ What is 2 + 2?<|im_end|> Ok(()) } + +/// Llama 3.2 has a chat template but no reasoning markers, so detection falls +/// through the FFI probe to the template probe and finds nothing. Shares the +/// ultravox load key so it joins an existing phase rather than adding one. +#[llama_test( + model_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "Llama-3.2-1B-Instruct-Q4_K_M.gguf" + ), + n_gpu_layers = 999, + load_mode = Auto, + n_ctx = 4096, + n_batch = 512, + n_ubatch = 512, + mmproj_source = HuggingFace( + "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF", + "mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf" + ), +)] +fn a_model_without_reasoning_markers_falls_through_to_the_template_probe( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let markers = fixture.model.reasoning_markers()?; + + assert!( + markers.is_none(), + "a chat template without thinking tags yields no reasoning markers; got: {markers:?}" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/sampling_and_constrained_decoding.rs b/llama-cpp-bindings-tests/tests/sampling_and_constrained_decoding.rs index 6fbe461b6..492af8999 100644 --- a/llama-cpp-bindings-tests/tests/sampling_and_constrained_decoding.rs +++ b/llama-cpp-bindings-tests/tests/sampling_and_constrained_decoding.rs @@ -26,8 +26,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 256, n_batch = 128, n_ubatch = 64, @@ -35,8 +34,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 256, n_batch = 128, n_ubatch = 64, @@ -44,8 +42,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 256, n_batch = 128, n_ubatch = 64, @@ -53,8 +50,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 256, n_batch = 128, n_ubatch = 64, @@ -85,8 +81,7 @@ fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -94,8 +89,7 @@ fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -103,8 +97,7 @@ fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -112,8 +105,7 @@ fn sample_returns_result_and_succeeds_with_valid_index(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -183,8 +175,7 @@ fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -192,8 +183,7 @@ fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -201,8 +191,7 @@ fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -210,8 +199,7 @@ fn grammar_sampler_constrains_output_to_yes_or_no(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -278,8 +266,7 @@ fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -287,8 +274,7 @@ fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -296,8 +282,7 @@ fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -305,8 +290,7 @@ fn json_schema_grammar_sampler_constrains_output_to_json(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -375,8 +359,7 @@ fn sample_with_grammar_produces_constrained_output_in_loop( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -384,8 +367,7 @@ fn sample_with_grammar_produces_constrained_output_in_loop( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -393,8 +375,7 @@ fn sample_with_grammar_produces_constrained_output_in_loop( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -402,8 +383,7 @@ fn sample_with_grammar_produces_constrained_output_in_loop( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -465,8 +445,7 @@ fn sample_without_grammar_produces_multiple_tokens(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -481,8 +460,7 @@ fn dry_sampler_with_model(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -501,8 +479,7 @@ fn dry_sampler_with_null_byte_in_seq_breakers_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -518,8 +495,7 @@ fn grammar_returns_sampler_for_valid_grammar(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -544,8 +520,7 @@ fn grammar_lazy_returns_sampler_for_valid_grammar_with_triggers( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -570,8 +545,7 @@ fn grammar_lazy_patterns_returns_sampler_for_valid_grammar_with_patterns( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -594,8 +568,7 @@ fn grammar_lazy_with_root_not_found_returns_error(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -620,8 +593,7 @@ fn grammar_lazy_with_null_byte_in_trigger_word_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -646,8 +618,7 @@ fn grammar_lazy_patterns_with_root_not_found_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -672,8 +643,7 @@ fn grammar_lazy_patterns_with_null_byte_in_pattern_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -701,8 +671,7 @@ fn grammar_lazy_patterns_with_malformed_regex_returns_invalid_trigger_pattern( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -718,8 +687,7 @@ fn llguidance_method_creates_sampler(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -735,8 +703,7 @@ fn logit_bias_with_empty_biases_succeeds(_fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -753,8 +720,7 @@ fn dry_sampler_with_root_not_found_grammar_does_not_apply( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -771,8 +737,7 @@ fn accept_many_iterates_over_borrowed_tokens(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -789,8 +754,7 @@ fn with_tokens_returns_self_after_accepting_each_token(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -806,8 +770,7 @@ fn accept_consumes_a_single_token(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -823,8 +786,7 @@ fn try_accept_returns_ok_for_a_valid_token(_fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -850,8 +812,7 @@ fn apply_runs_sampler_over_token_data_array(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 2048, n_ubatch = 512, @@ -877,8 +838,7 @@ fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -886,8 +846,7 @@ fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -895,8 +854,7 @@ fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -904,8 +862,7 @@ fn sample_returns_token_after_decode(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1025,8 +982,7 @@ fn raw_prompt_completion_with_timing(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128, @@ -1034,8 +990,7 @@ fn raw_prompt_completion_with_timing(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128, @@ -1043,8 +998,7 @@ fn raw_prompt_completion_with_timing(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128, @@ -1052,8 +1006,7 @@ fn raw_prompt_completion_with_timing(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128, @@ -1153,8 +1106,7 @@ fn chat_inference_produces_coherent_output(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1162,8 +1114,7 @@ fn chat_inference_produces_coherent_output(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1171,8 +1122,7 @@ fn chat_inference_produces_coherent_output(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1180,8 +1130,7 @@ fn chat_inference_produces_coherent_output(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1269,8 +1218,7 @@ const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1278,8 +1226,7 @@ const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1287,8 +1234,7 @@ const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1296,8 +1242,7 @@ const LARK_GRAMMAR: &str = r#"start: "yes" | "no""#; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1313,8 +1258,7 @@ fn creates_sampler_with_valid_json_schema(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1322,8 +1266,7 @@ fn creates_sampler_with_valid_json_schema(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1331,8 +1274,7 @@ fn creates_sampler_with_valid_json_schema(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1340,8 +1282,7 @@ fn creates_sampler_with_valid_json_schema(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1357,8 +1298,7 @@ fn creates_sampler_with_valid_regex_grammar(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1366,8 +1306,7 @@ fn creates_sampler_with_valid_regex_grammar(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1375,8 +1314,7 @@ fn creates_sampler_with_valid_regex_grammar(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1384,8 +1322,7 @@ fn creates_sampler_with_valid_regex_grammar(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1401,8 +1338,7 @@ fn creates_sampler_with_valid_lark_grammar(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1410,8 +1346,7 @@ fn creates_sampler_with_valid_lark_grammar(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1419,8 +1354,7 @@ fn creates_sampler_with_valid_lark_grammar(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1428,8 +1362,7 @@ fn creates_sampler_with_valid_lark_grammar(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1444,8 +1377,7 @@ fn returns_error_for_unknown_grammar_kind(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1453,8 +1385,7 @@ fn returns_error_for_unknown_grammar_kind(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1462,8 +1393,7 @@ fn returns_error_for_unknown_grammar_kind(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1471,8 +1401,7 @@ fn returns_error_for_unknown_grammar_kind(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1487,8 +1416,7 @@ fn returns_error_for_malformed_json_schema(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1496,8 +1424,7 @@ fn returns_error_for_malformed_json_schema(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1505,8 +1432,7 @@ fn returns_error_for_malformed_json_schema(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1514,8 +1440,7 @@ fn returns_error_for_malformed_json_schema(fixture: &LlamaFixture<'_>) -> Result #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1530,8 +1455,7 @@ fn returns_error_for_malformed_regex(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1539,8 +1463,7 @@ fn returns_error_for_malformed_regex(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1548,8 +1471,7 @@ fn returns_error_for_malformed_regex(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1557,8 +1479,7 @@ fn returns_error_for_malformed_regex(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1578,8 +1499,7 @@ fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1587,8 +1507,7 @@ fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1596,8 +1515,7 @@ fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1605,8 +1523,7 @@ fn name_callback_returns_llguidance(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1626,8 +1543,7 @@ fn clone_via_ffi_creates_independent_sampler(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1635,8 +1551,7 @@ fn clone_via_ffi_creates_independent_sampler(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1644,8 +1559,7 @@ fn clone_via_ffi_creates_independent_sampler(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1653,8 +1567,7 @@ fn clone_via_ffi_creates_independent_sampler(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1689,8 +1602,7 @@ fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1698,8 +1610,7 @@ fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1707,8 +1618,7 @@ fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1716,8 +1626,7 @@ fn samples_token_constrained_by_grammar(fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1734,8 +1643,7 @@ fn accept_invalid_token_id_does_not_panic(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1743,8 +1651,7 @@ fn accept_invalid_token_id_does_not_panic(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1752,8 +1659,7 @@ fn accept_invalid_token_id_does_not_panic(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1761,8 +1667,7 @@ fn accept_invalid_token_id_does_not_panic(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1779,8 +1684,7 @@ fn approximate_tok_env_returns_same_arc_across_calls(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1788,8 +1692,7 @@ fn approximate_tok_env_returns_same_arc_across_calls(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1797,8 +1700,7 @@ fn approximate_tok_env_returns_same_arc_across_calls(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1806,8 +1708,7 @@ fn approximate_tok_env_returns_same_arc_across_calls(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1827,8 +1728,7 @@ fn approximate_tok_env_drives_consistent_grammar_constraint( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1836,8 +1736,7 @@ fn approximate_tok_env_drives_consistent_grammar_constraint( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1845,8 +1744,7 @@ fn approximate_tok_env_drives_consistent_grammar_constraint( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1854,8 +1752,7 @@ fn approximate_tok_env_drives_consistent_grammar_constraint( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1884,8 +1781,7 @@ fn apply_through_chain_during_sample_does_not_panic(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1893,8 +1789,7 @@ fn apply_through_chain_during_sample_does_not_panic(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1902,8 +1797,7 @@ fn apply_through_chain_during_sample_does_not_panic(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1911,8 +1805,7 @@ fn apply_through_chain_during_sample_does_not_panic(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 512, n_ubatch = 128, @@ -1936,8 +1829,7 @@ fn reset_clears_sampler_state(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1945,8 +1837,7 @@ fn reset_clears_sampler_state(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1954,8 +1845,7 @@ fn reset_clears_sampler_state(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1963,8 +1853,7 @@ fn reset_clears_sampler_state(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1981,8 +1870,7 @@ fn classifier_starts_in_pending_section_for_default_fixture( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1990,8 +1878,7 @@ fn classifier_starts_in_pending_section_for_default_fixture( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1999,8 +1886,7 @@ fn classifier_starts_in_pending_section_for_default_fixture( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2008,8 +1894,7 @@ fn classifier_starts_in_pending_section_for_default_fixture( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2026,8 +1911,7 @@ fn classifier_construction_is_idempotent_across_calls(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2035,8 +1919,7 @@ fn classifier_construction_is_idempotent_across_calls(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2044,8 +1927,7 @@ fn classifier_construction_is_idempotent_across_calls(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2053,8 +1935,7 @@ fn classifier_construction_is_idempotent_across_calls(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2081,8 +1962,7 @@ fn ingest_with_no_markers_emits_undeterminable_with_visible_and_raw_piece( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2090,8 +1970,7 @@ fn ingest_with_no_markers_emits_undeterminable_with_visible_and_raw_piece( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2099,8 +1978,7 @@ fn ingest_with_no_markers_emits_undeterminable_with_visible_and_raw_piece( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2108,8 +1986,7 @@ fn ingest_with_no_markers_emits_undeterminable_with_visible_and_raw_piece( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2130,8 +2007,7 @@ fn ingest_with_no_markers_decodes_each_token_independently( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2139,8 +2015,7 @@ fn ingest_with_no_markers_decodes_each_token_independently( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2148,8 +2023,7 @@ fn ingest_with_no_markers_decodes_each_token_independently( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2157,8 +2031,7 @@ fn ingest_with_no_markers_decodes_each_token_independently( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2179,8 +2052,7 @@ fn ingest_prompt_token_with_no_markers_is_a_noop(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2188,8 +2060,7 @@ fn ingest_prompt_token_with_no_markers_is_a_noop(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2197,8 +2068,7 @@ fn ingest_prompt_token_with_no_markers_is_a_noop(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2206,8 +2076,7 @@ fn ingest_prompt_token_with_no_markers_is_a_noop(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2229,8 +2098,7 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2238,8 +2106,7 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2247,8 +2114,7 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2256,8 +2122,7 @@ fn feed_prompt_to_batch_increments_pending_prompt_tokens(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2279,8 +2144,7 @@ fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2288,8 +2152,7 @@ fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2297,8 +2160,7 @@ fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2306,8 +2168,7 @@ fn feed_prompt_sequence_to_batch_stages_all_tokens(fixture: &LlamaFixture<'_>) - #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2334,8 +2195,7 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2343,8 +2203,7 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2352,8 +2211,7 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2361,8 +2219,7 @@ fn commit_prompt_tokens_promotes_pending_count_to_usage_and_clears( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2388,8 +2245,7 @@ fn discard_pending_prompt_tokens_clears_count_without_recording_usage( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2397,8 +2253,7 @@ fn discard_pending_prompt_tokens_clears_count_without_recording_usage( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2406,8 +2261,7 @@ fn discard_pending_prompt_tokens_clears_count_without_recording_usage( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2415,8 +2269,7 @@ fn discard_pending_prompt_tokens_clears_count_without_recording_usage( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -2429,3 +2282,68 @@ fn diagnose_tool_call_synthetic_renders_returns_a_pair_of_strings( let _ = right; Ok(()) } + +/// Once a regex grammar is fully satisfied the parser reports completion, and +/// the sampler must stop constraining rather than error. Sampling continues +/// past the match so at least one post-completion mask is computed. +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Auto, + n_ctx = 512, + n_batch = 512, + n_ubatch = 128, +)] +fn a_satisfied_grammar_stops_constraining(fixture: &LlamaFixture<'_>) -> Result<()> { + let model = fixture.model; + + let mut ctx = LlamaContext::from_model( + model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + + let tokens_list = model.str_to_token("Answer with one word:\n", AddBos::Always)?; + let mut batch = LlamaBatch::new(512, 1)?; + let last_index = i32::try_from(tokens_list.len())? - 1; + + for (index, token) in (0_i32..).zip(&tokens_list) { + batch.add( + &SampledToken::Content(*token), + index, + &[0], + index == last_index, + )?; + } + + ctx.decode(&mut batch)?; + + let llg_sampler = LlamaSampler::llguidance(model, "regex", r"yes")?; + let mut sampler = LlamaSampler::chain_simple([llg_sampler, LlamaSampler::greedy()]); + + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let mut generated = String::new(); + + // Samples beyond the grammar's match so the sampler is asked for a mask + // after the parser has already accepted the whole pattern. + for n_cur in batch.n_tokens()..batch.n_tokens() + 8 { + let token = SampledToken::Content(sampler.sample(&ctx, batch.n_tokens() - 1)?); + + if model.is_eog_token(&token) { + break; + } + + generated.push_str(&model.token_to_piece(&token, &mut decoder, true, None)?); + + batch.clear(); + batch.add(&token, n_cur, &[0], true)?; + ctx.decode(&mut batch)?; + } + + assert!( + generated.starts_with("yes"), + "the grammar must constrain the output until it is satisfied; got: {generated:?}" + ); + + Ok(()) +} diff --git a/llama-cpp-bindings-tests/tests/vocabulary_and_metadata.rs b/llama-cpp-bindings-tests/tests/vocabulary_and_metadata.rs index 81f55876f..a60cfd828 100644 --- a/llama-cpp-bindings-tests/tests/vocabulary_and_metadata.rs +++ b/llama-cpp-bindings-tests/tests/vocabulary_and_metadata.rs @@ -14,8 +14,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -23,8 +22,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -32,8 +30,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -41,8 +38,7 @@ use llama_cpp_test_harness::llama_test; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -61,8 +57,7 @@ fn model_loads_with_valid_metadata(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -70,8 +65,7 @@ fn model_loads_with_valid_metadata(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -79,8 +73,7 @@ fn model_loads_with_valid_metadata(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -88,8 +81,7 @@ fn model_loads_with_valid_metadata(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -102,8 +94,7 @@ fn n_layer_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -111,8 +102,7 @@ fn n_layer_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -120,8 +110,7 @@ fn n_layer_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -129,8 +118,7 @@ fn n_layer_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -143,8 +131,7 @@ fn n_head_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -152,8 +139,7 @@ fn n_head_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -161,8 +147,7 @@ fn n_head_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -170,8 +155,7 @@ fn n_head_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -184,8 +168,7 @@ fn n_head_kv_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -193,8 +176,7 @@ fn n_head_kv_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -202,8 +184,7 @@ fn n_head_kv_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -211,8 +192,7 @@ fn n_head_kv_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -225,8 +205,7 @@ fn model_size_returns_nonzero(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -234,8 +213,7 @@ fn model_size_returns_nonzero(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -243,8 +221,7 @@ fn model_size_returns_nonzero(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -252,8 +229,7 @@ fn model_size_returns_nonzero(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -266,8 +242,7 @@ fn is_recurrent_returns_false_for_transformer(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -275,8 +250,7 @@ fn is_recurrent_returns_false_for_transformer(fixture: &LlamaFixture<'_>) -> Res #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -292,8 +266,7 @@ fn is_hybrid_returns_false_for_non_hybrid_default_models(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -301,8 +274,7 @@ fn is_hybrid_returns_false_for_non_hybrid_default_models(fixture: &LlamaFixture< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -318,8 +290,7 @@ fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -327,8 +298,7 @@ fn is_hybrid_returns_true_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -351,8 +321,7 @@ fn rope_type_returns_a_known_variant_for_rope_carrying_default_models( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -360,8 +329,7 @@ fn rope_type_returns_a_known_variant_for_rope_carrying_default_models( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -378,8 +346,7 @@ fn rope_type_returns_none_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -387,8 +354,7 @@ fn rope_type_returns_none_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -396,8 +362,7 @@ fn rope_type_returns_none_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -405,8 +370,7 @@ fn rope_type_returns_none_for_qwen_default_models(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -424,8 +388,7 @@ fn vocab_type_returns_a_known_variant(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -433,8 +396,7 @@ fn vocab_type_returns_a_known_variant(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -442,8 +404,7 @@ fn vocab_type_returns_a_known_variant(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -451,8 +412,7 @@ fn vocab_type_returns_a_known_variant(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -465,8 +425,7 @@ fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -474,8 +433,7 @@ fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -483,8 +441,7 @@ fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -492,8 +449,7 @@ fn meta_count_returns_positive(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -507,8 +463,7 @@ fn meta_key_by_index_returns_valid_key(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -516,8 +471,7 @@ fn meta_key_by_index_returns_valid_key(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -525,8 +479,7 @@ fn meta_key_by_index_returns_valid_key(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -534,8 +487,7 @@ fn meta_key_by_index_returns_valid_key(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -549,8 +501,7 @@ fn meta_val_str_by_index_returns_valid_value(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -558,8 +509,7 @@ fn meta_val_str_by_index_returns_valid_value(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -567,8 +517,7 @@ fn meta_val_str_by_index_returns_valid_value(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -576,8 +525,7 @@ fn meta_val_str_by_index_returns_valid_value(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -591,8 +539,7 @@ fn meta_key_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -600,8 +547,7 @@ fn meta_key_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -609,8 +555,7 @@ fn meta_key_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -618,8 +563,7 @@ fn meta_key_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) -> R #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -633,8 +577,7 @@ fn meta_val_str_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -642,8 +585,7 @@ fn meta_val_str_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -651,8 +593,7 @@ fn meta_val_str_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -660,8 +601,7 @@ fn meta_val_str_by_index_out_of_range_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -677,8 +617,7 @@ fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -686,8 +625,7 @@ fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -695,8 +633,7 @@ fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -704,8 +641,7 @@ fn meta_val_str_returns_value_for_known_key(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -726,8 +662,7 @@ fn meta_val_str_with_long_value_triggers_buffer_resize(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -735,8 +670,7 @@ fn meta_val_str_with_long_value_triggers_buffer_resize(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -744,8 +678,7 @@ fn meta_val_str_with_long_value_triggers_buffer_resize(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -753,8 +686,7 @@ fn meta_val_str_with_long_value_triggers_buffer_resize(fixture: &LlamaFixture<'_ #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -768,8 +700,7 @@ fn meta_val_str_with_null_byte_in_key_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -777,8 +708,7 @@ fn meta_val_str_with_null_byte_in_key_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -786,8 +716,7 @@ fn meta_val_str_with_null_byte_in_key_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -795,8 +724,7 @@ fn meta_val_str_with_null_byte_in_key_returns_error(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -829,8 +757,7 @@ fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -838,8 +765,7 @@ fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -847,8 +773,7 @@ fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -856,8 +781,7 @@ fn fit_params_succeeds_with_test_model(fixture: &LlamaFixture<'_>) -> Result<()> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -875,8 +799,7 @@ fn special_tokens_exist(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -884,8 +807,7 @@ fn special_tokens_exist(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -893,8 +815,7 @@ fn special_tokens_exist(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -902,8 +823,7 @@ fn special_tokens_exist(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -917,8 +837,7 @@ fn token_nl_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -926,8 +845,7 @@ fn token_nl_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -935,8 +853,7 @@ fn token_nl_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -944,8 +861,7 @@ fn token_nl_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -960,8 +876,7 @@ fn is_eog_token_classifies_reasoning_variant(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -969,8 +884,7 @@ fn is_eog_token_classifies_reasoning_variant(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -978,8 +892,7 @@ fn is_eog_token_classifies_reasoning_variant(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -987,8 +900,7 @@ fn is_eog_token_classifies_reasoning_variant(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1003,8 +915,7 @@ fn is_eog_token_classifies_tool_call_variant(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1012,8 +923,7 @@ fn is_eog_token_classifies_tool_call_variant(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1021,8 +931,7 @@ fn is_eog_token_classifies_tool_call_variant(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1030,8 +939,7 @@ fn is_eog_token_classifies_tool_call_variant(fixture: &LlamaFixture<'_>) -> Resu #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1046,8 +954,7 @@ fn is_eog_token_classifies_undeterminable_variant(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1055,8 +962,7 @@ fn is_eog_token_classifies_undeterminable_variant(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1064,8 +970,7 @@ fn is_eog_token_classifies_undeterminable_variant(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1073,8 +978,7 @@ fn is_eog_token_classifies_undeterminable_variant(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1098,8 +1002,7 @@ fn decode_start_token_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1107,8 +1010,7 @@ fn decode_start_token_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1116,8 +1018,7 @@ fn decode_start_token_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1125,8 +1026,7 @@ fn decode_start_token_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1150,8 +1050,7 @@ fn token_sep_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1159,8 +1058,7 @@ fn token_sep_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1168,8 +1066,7 @@ fn token_sep_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1177,8 +1074,7 @@ fn token_sep_returns_valid_token(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1198,8 +1094,7 @@ fn token_attr_returns_attrs_for_bos(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1207,8 +1102,7 @@ fn token_attr_returns_attrs_for_bos(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1216,8 +1110,7 @@ fn token_attr_returns_attrs_for_bos(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1225,8 +1118,7 @@ fn token_attr_returns_attrs_for_bos(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1247,8 +1139,7 @@ fn str_to_token_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1256,8 +1147,7 @@ fn str_to_token_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1265,8 +1155,7 @@ fn str_to_token_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1274,8 +1163,7 @@ fn str_to_token_roundtrip(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1300,8 +1188,7 @@ fn str_to_token_grows_buffer_when_initial_estimation_too_small( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1309,8 +1196,7 @@ fn str_to_token_grows_buffer_when_initial_estimation_too_small( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1318,8 +1204,7 @@ fn str_to_token_grows_buffer_when_initial_estimation_too_small( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1327,8 +1212,7 @@ fn str_to_token_grows_buffer_when_initial_estimation_too_small( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1346,8 +1230,7 @@ fn str_to_token_with_add_bos_never(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1355,8 +1238,7 @@ fn str_to_token_with_add_bos_never(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1364,8 +1246,7 @@ fn str_to_token_with_add_bos_never(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1373,8 +1254,7 @@ fn str_to_token_with_add_bos_never(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1397,8 +1277,7 @@ fn str_to_token_with_many_tokens_triggers_buffer_resize(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1406,8 +1285,7 @@ fn str_to_token_with_many_tokens_triggers_buffer_resize(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1415,8 +1293,7 @@ fn str_to_token_with_many_tokens_triggers_buffer_resize(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1424,8 +1301,7 @@ fn str_to_token_with_many_tokens_triggers_buffer_resize(fixture: &LlamaFixture<' #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1442,8 +1318,7 @@ fn token_to_piece_bytes_returns_bytes_for_known_token(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1451,8 +1326,7 @@ fn token_to_piece_bytes_returns_bytes_for_known_token(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1460,8 +1334,7 @@ fn token_to_piece_bytes_returns_bytes_for_known_token(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1469,8 +1342,7 @@ fn token_to_piece_bytes_returns_bytes_for_known_token(fixture: &LlamaFixture<'_> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1491,8 +1363,7 @@ fn token_to_piece_handles_large_token_requiring_buffer_resize( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1500,8 +1371,7 @@ fn token_to_piece_handles_large_token_requiring_buffer_resize( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1509,8 +1379,7 @@ fn token_to_piece_handles_large_token_requiring_buffer_resize( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1518,8 +1387,7 @@ fn token_to_piece_handles_large_token_requiring_buffer_resize( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1543,8 +1411,7 @@ fn token_to_piece_bytes_insufficient_buffer_returns_error( #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1552,8 +1419,7 @@ fn token_to_piece_bytes_insufficient_buffer_returns_error( #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1561,8 +1427,7 @@ fn token_to_piece_bytes_insufficient_buffer_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1570,8 +1435,7 @@ fn token_to_piece_bytes_insufficient_buffer_returns_error( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1594,8 +1458,7 @@ fn token_to_piece_with_lstrip(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1603,8 +1466,7 @@ fn token_to_piece_with_lstrip(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1612,8 +1474,7 @@ fn token_to_piece_with_lstrip(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1621,8 +1482,7 @@ fn token_to_piece_with_lstrip(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1646,8 +1506,7 @@ fn token_to_piece_decodes_reasoning_variant(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1655,8 +1514,7 @@ fn token_to_piece_decodes_reasoning_variant(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1664,8 +1522,7 @@ fn token_to_piece_decodes_reasoning_variant(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1673,8 +1530,7 @@ fn token_to_piece_decodes_reasoning_variant(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1694,8 +1550,7 @@ fn token_to_piece_decodes_tool_call_variant(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1703,8 +1558,7 @@ fn token_to_piece_decodes_tool_call_variant(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1712,8 +1566,7 @@ fn token_to_piece_decodes_tool_call_variant(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1721,8 +1574,7 @@ fn token_to_piece_decodes_tool_call_variant(fixture: &LlamaFixture<'_>) -> Resul #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1746,8 +1598,7 @@ fn token_to_piece_decodes_undeterminable_variant(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1755,8 +1606,7 @@ fn token_to_piece_decodes_undeterminable_variant(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1764,8 +1614,7 @@ fn token_to_piece_decodes_undeterminable_variant(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1773,8 +1622,7 @@ fn token_to_piece_decodes_undeterminable_variant(fixture: &LlamaFixture<'_>) -> #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1799,8 +1647,7 @@ fn tokens_iterator_produces_valid_entries(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF", "DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1808,8 +1655,7 @@ fn tokens_iterator_produces_valid_entries(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/GLM-4.7-Flash-GGUF", "GLM-4.7-Flash-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1817,8 +1663,7 @@ fn tokens_iterator_produces_valid_entries(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1826,8 +1671,7 @@ fn tokens_iterator_produces_valid_entries(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -1844,8 +1688,7 @@ fn n_vocab_matches_tokens_iterator_count(fixture: &LlamaFixture<'_>) -> Result<( #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128 @@ -1862,8 +1705,7 @@ fn debug_format_includes_struct_name_and_model_field(fixture: &LlamaFixture<'_>) #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 2048, n_batch = 512, n_ubatch = 128 diff --git a/llama-cpp-bindings-types/src/lib.rs b/llama-cpp-bindings-types/src/lib.rs index 194b43cae..e69126a87 100644 --- a/llama-cpp-bindings-types/src/lib.rs +++ b/llama-cpp-bindings-types/src/lib.rs @@ -9,6 +9,8 @@ pub mod key_value_xml_tags_shape; pub mod paired_quote_shape; pub mod parsed_chat_message; pub mod parsed_tool_call; +pub mod probed_reasoning_markers; +pub mod reasoning_close_match; pub mod reasoning_markers; pub mod token_usage; pub mod token_usage_error; @@ -24,6 +26,8 @@ pub use key_value_xml_tags_shape::KeyValueXmlTagsShape; pub use paired_quote_shape::PairedQuoteShape; pub use parsed_chat_message::ParsedChatMessage; pub use parsed_tool_call::ParsedToolCall; +pub use probed_reasoning_markers::ProbedReasoningMarkers; +pub use reasoning_close_match::ReasoningCloseMatch; pub use reasoning_markers::ReasoningMarkers; pub use token_usage::TokenUsage; pub use token_usage_error::TokenUsageError; diff --git a/llama-cpp-bindings-types/src/probed_reasoning_markers.rs b/llama-cpp-bindings-types/src/probed_reasoning_markers.rs new file mode 100644 index 000000000..d89b3bef4 --- /dev/null +++ b/llama-cpp-bindings-types/src/probed_reasoning_markers.rs @@ -0,0 +1,40 @@ +use crate::reasoning_markers::ReasoningMarkers; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProbedReasoningMarkers { + pub open: String, + pub close: String, +} + +impl From for ReasoningMarkers { + fn from(ProbedReasoningMarkers { open, close }: ProbedReasoningMarkers) -> Self { + Self { + open, + closes: vec![close], + } + } +} + +#[cfg(test)] +mod tests { + use crate::reasoning_markers::ReasoningMarkers; + + use super::ProbedReasoningMarkers; + + #[test] + fn a_probed_pair_widens_to_a_single_candidate_list() { + let widened: ReasoningMarkers = ProbedReasoningMarkers { + open: "".to_owned(), + close: "".to_owned(), + } + .into(); + + assert_eq!( + widened, + ReasoningMarkers { + open: "".to_owned(), + closes: vec!["".to_owned()], + } + ); + } +} diff --git a/llama-cpp-bindings-types/src/reasoning_close_match.rs b/llama-cpp-bindings-types/src/reasoning_close_match.rs new file mode 100644 index 000000000..e6f9742b2 --- /dev/null +++ b/llama-cpp-bindings-types/src/reasoning_close_match.rs @@ -0,0 +1,5 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ReasoningCloseMatch { + pub offset: usize, + pub length: usize, +} diff --git a/llama-cpp-bindings-types/src/reasoning_markers.rs b/llama-cpp-bindings-types/src/reasoning_markers.rs index 02d7586a6..6570a3be2 100644 --- a/llama-cpp-bindings-types/src/reasoning_markers.rs +++ b/llama-cpp-bindings-types/src/reasoning_markers.rs @@ -1,5 +1,97 @@ +use crate::reasoning_close_match::ReasoningCloseMatch; + #[derive(Clone, Debug, Eq, PartialEq)] pub struct ReasoningMarkers { pub open: String, - pub close: String, + pub closes: Vec, +} + +impl ReasoningMarkers { + #[must_use] + pub fn is_usable(&self) -> bool { + !self.open.is_empty() && self.closes.iter().any(|close| !close.is_empty()) + } + + #[must_use] + pub fn find_earliest_close(&self, haystack: &str) -> Option { + self.closes + .iter() + .filter(|close| !close.is_empty()) + .filter_map(|close| { + haystack + .find(close.as_str()) + .map(|offset| ReasoningCloseMatch { + offset, + length: close.len(), + }) + }) + .min_by_key(|candidate| (candidate.offset, usize::MAX - candidate.length)) + } +} + +#[cfg(test)] +mod tests { + use crate::reasoning_close_match::ReasoningCloseMatch; + + use super::ReasoningMarkers; + + fn markers(closes: &[&str]) -> ReasoningMarkers { + ReasoningMarkers { + open: "".to_owned(), + closes: closes.iter().map(|close| (*close).to_owned()).collect(), + } + } + + #[test] + fn earliest_close_wins_over_a_later_one() { + let found = markers(&["", ""]) + .find_earliest_close("reasoningmore"); + + assert_eq!( + found, + Some(ReasoningCloseMatch { + offset: 9, + length: "".len() + }) + ); + } + + #[test] + fn longest_candidate_wins_at_the_same_offset() { + let found = markers(&[""]).find_earliest_close("abc"); + + assert_eq!( + found, + Some(ReasoningCloseMatch { + offset: 3, + length: "".len() + }) + ); + } + + #[test] + fn absent_candidates_yield_no_match() { + assert_eq!( + markers(&[""]).find_earliest_close("no marker here"), + None + ); + } + + #[test] + fn empty_candidates_are_ignored() { + assert_eq!(markers(&[""]).find_earliest_close("anything"), None); + } + + #[test] + fn is_usable_requires_an_open_and_a_non_empty_close() { + assert!(markers(&[""]).is_usable()); + assert!(!markers(&[""]).is_usable()); + assert!( + !ReasoningMarkers { + open: String::new(), + closes: vec!["".to_owned()], + } + .is_usable() + ); + } } diff --git a/llama-cpp-bindings/src/extract_reasoning_markers_from_probe_renders.rs b/llama-cpp-bindings/src/extract_reasoning_markers_from_probe_renders.rs index 9cc099957..b2b2e8add 100644 --- a/llama-cpp-bindings/src/extract_reasoning_markers_from_probe_renders.rs +++ b/llama-cpp-bindings/src/extract_reasoning_markers_from_probe_renders.rs @@ -1,6 +1,6 @@ use serde_json::json; -use crate::ReasoningMarkers; +use crate::ProbedReasoningMarkers; const REASON_PROBE: &str = "__PADDLER_REASON_PROBE_3F4A8C__"; const RESPONSE_PROBE: &str = "__PADDLER_RESPONSE_PROBE_3F4A8C__"; @@ -60,7 +60,7 @@ fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool { pub fn extract_reasoning_markers_from_probe_renders( plain_render: &str, chunked_render: &str, -) -> Option { +) -> Option { let plain = plain_render.as_bytes(); let chunked = chunked_render.as_bytes(); @@ -86,10 +86,6 @@ pub fn extract_reasoning_markers_from_probe_renders( common_suffix += 1; } - if common_prefix + common_suffix > chunked_size { - return None; - } - let diff = &chunked[common_prefix..chunked_size - common_suffix]; let reason_pos = find_subslice(diff, REASON_PROBE.as_bytes())?; @@ -112,7 +108,7 @@ pub fn extract_reasoning_markers_from_probe_renders( return None; } - Some(ReasoningMarkers { open, close }) + Some(ProbedReasoningMarkers { open, close }) } #[cfg(test)] @@ -157,4 +153,15 @@ mod tests { assert!(extract_reasoning_markers_from_probe_renders(&plain, &chunked).is_none()); } + + #[test] + fn returns_none_when_the_close_marker_would_be_empty() { + let plain = format!("PREFIX{RESPONSE_PROBE}SUFFIX"); + let chunked = format!("PREFIX{REASON_PROBE}{RESPONSE_PROBE}SUFFIX"); + + assert!( + extract_reasoning_markers_from_probe_renders(&plain, &chunked).is_none(), + "a render with nothing between the reasoning and response probes has no close marker" + ); + } } diff --git a/llama-cpp-bindings/src/lib.rs b/llama-cpp-bindings/src/lib.rs index 0ff8697f1..ba296c5c5 100644 --- a/llama-cpp-bindings/src/lib.rs +++ b/llama-cpp-bindings/src/lib.rs @@ -77,8 +77,8 @@ pub use llama_backend_device::{LlamaBackendDevice, list_llama_ggml_backend_devic pub use llama_backend_device_type::LlamaBackendDeviceType; pub use llama_cpp_bindings_types::{ BracketedJsonShape, KeyValueXmlTagsShape, PairedQuoteShape, ParsedChatMessage, ParsedToolCall, - ReasoningMarkers, TokenUsage, TokenUsageError, ToolCallArgsShape, ToolCallArguments, - ToolCallMarkers, ToolCallValueQuote, XmlTagsShape, + ProbedReasoningMarkers, ReasoningCloseMatch, ReasoningMarkers, TokenUsage, TokenUsageError, + ToolCallArgsShape, ToolCallArguments, ToolCallMarkers, ToolCallValueQuote, XmlTagsShape, }; pub use raw_chat_message::RawChatMessage; pub use sampled_token::SampledToken; diff --git a/llama-cpp-bindings/src/llguidance_sampler.rs b/llama-cpp-bindings/src/llguidance_sampler.rs index 9cd28801c..1cd4ca90a 100644 --- a/llama-cpp-bindings/src/llguidance_sampler.rs +++ b/llama-cpp-bindings/src/llguidance_sampler.rs @@ -105,6 +105,8 @@ static mut LLG_SAMPLER_I: llama_cpp_bindings_sys::llama_sampler_i = backend_accept: None, backend_apply: None, backend_set_input: None, + backend_reset: None, + copy_state: None, }; /// # Errors diff --git a/llama-cpp-bindings/src/model.rs b/llama-cpp-bindings/src/model.rs index 49f97f247..49cf52f09 100644 --- a/llama-cpp-bindings/src/model.rs +++ b/llama-cpp-bindings/src/model.rs @@ -1,8 +1,10 @@ pub mod add_bos; pub mod llama_chat_message; pub mod llama_chat_template; +pub mod llama_load_mode_parse_error; pub mod llama_lora_adapter; pub mod llama_split_mode_parse_error; +pub mod load_mode; pub mod params; pub mod rope_type; pub mod split_mode; @@ -24,6 +26,7 @@ use toktrie::TokTrie; use llama_cpp_bindings_types::ParsedChatMessage; use llama_cpp_bindings_types::ParsedToolCall; +use llama_cpp_bindings_types::ProbedReasoningMarkers; use llama_cpp_bindings_types::ReasoningMarkers; use llama_cpp_bindings_types::ToolCallArguments; use llama_cpp_bindings_types::ToolCallMarkers; @@ -253,18 +256,6 @@ unsafe fn parsed_chat_free_status_to_result( } } -fn reasoning_markers_from_marker_pair( - open: Option, - close: Option, -) -> Option { - match (open, close) { - (Some(open), Some(close)) if !open.is_empty() && !close.is_empty() => { - Some(ReasoningMarkers { open, close }) - } - _ => None, - } -} - fn outcome_from_via_ffi_result( via_ffi_result: Result, tools_json: &str, @@ -789,8 +780,7 @@ impl LlamaModel { /// # Errors /// Returns [`MarkerDetectionError`] when any underlying FFI call fails. pub fn streaming_markers(&self) -> Result { - let (reasoning_open_str, reasoning_close_str) = - invoke_detect_reasoning_markers(self.model.as_ptr())?; + let detected_reasoning = invoke_detect_reasoning_markers(self.model.as_ptr())?; let tool_call_haystack = invoke_compute_tool_call_haystack(self.model.as_ptr())?; @@ -809,10 +799,11 @@ impl LlamaModel { self.resolve_tool_call_marker_strings(autoparser_open, autoparser_close)?; Ok(StreamingMarkers { - reasoning_open: self.tokenize_marker(reasoning_open_str.as_deref())?, - reasoning_close: self.tokenize_marker(reasoning_close_str.as_deref())?, - tool_call_open: self.tokenize_marker(resolved_tool_call_markers.open.as_deref())?, - tool_call_close: self.tokenize_marker(resolved_tool_call_markers.close.as_deref())?, + reasoning_open: self.tokenize_markers(detected_reasoning.open.as_deref())?, + reasoning_close: self + .tokenize_markers(detected_reasoning.closes.iter().map(String::as_str))?, + tool_call_open: self.tokenize_markers(resolved_tool_call_markers.open.as_deref())?, + tool_call_close: self.tokenize_markers(resolved_tool_call_markers.close.as_deref())?, }) } @@ -850,13 +841,13 @@ impl LlamaModel { /// # Errors /// Returns [`MarkerDetectionError`] when the underlying FFI call fails. pub fn reasoning_markers(&self) -> Result, MarkerDetectionError> { - let (open, close) = invoke_detect_reasoning_markers(self.model.as_ptr())?; + let detected = invoke_detect_reasoning_markers(self.model.as_ptr())?; - if let Some(markers) = reasoning_markers_from_marker_pair(open, close) { + if let Some(markers) = detected.into_reasoning_markers() { return Ok(Some(markers)); } - detect_reasoning_markers_via_template_probe(self.model.as_ptr()) + Ok(detect_reasoning_markers_via_template_probe(self.model.as_ptr())?.map(Into::into)) } /// # Errors @@ -877,13 +868,7 @@ impl LlamaModel { /// # Errors /// Returns [`StringToTokenError`] when a present, non-empty marker string /// fails to tokenise. - fn tokenize_marker( - &self, - marker: Option<&str>, - ) -> Result>, StringToTokenError> { - let Some(marker) = marker else { - return Ok(None); - }; + fn tokenize_marker(&self, marker: &str) -> Result>, StringToTokenError> { let marker = marker.trim(); if marker.is_empty() { return Ok(None); @@ -896,6 +881,16 @@ impl LlamaModel { } } + fn tokenize_markers<'marker>( + &self, + markers: impl IntoIterator, + ) -> Result>, StringToTokenError> { + markers + .into_iter() + .filter_map(|marker| self.tokenize_marker(marker).transpose()) + .collect() + } + /// # Errors /// /// Returns [`ParseChatMessageError`] when `tools_json` is not valid JSON, @@ -1422,12 +1417,12 @@ fn split_reasoning_prefix( }; let after_open = &input[open_pos + reasoning_markers.open.len()..]; - let Some(close_offset) = after_open.find(&reasoning_markers.close) else { + let Some(close) = reasoning_markers.find_earliest_close(after_open) else { return content_only(); }; - let reasoning = after_open[..close_offset].to_owned(); - let after_close = &after_open[close_offset + reasoning_markers.close.len()..]; + let reasoning = after_open[..close.offset].to_owned(); + let after_close = &after_open[close.offset + close.length..]; ReasoningSplit { reasoning, @@ -1448,30 +1443,115 @@ fn synthesize_missing_tool_call_ids(tool_calls: &mut [ParsedToolCall]) { } } -// SAFETY: `out_open`, `out_close`, and `out_error` must be the pointers -// populated by the preceding `llama_rs_detect_reasoning_markers` call (or null). -// `out_open`/`out_close` are read but not freed here; `out_error` is freed only -// in the CXX-exception arm, mirroring the conditional cleanup in the caller. -unsafe fn detect_reasoning_markers_status_to_result( +struct DetectedReasoningMarkers { + open: Option, + closes: Vec, +} + +impl DetectedReasoningMarkers { + fn into_reasoning_markers(self) -> Option { + let markers = ReasoningMarkers { + open: self.open?, + closes: self + .closes + .into_iter() + .filter(|close| !close.is_empty()) + .collect(), + }; + + markers.is_usable().then_some(markers) + } +} + +/// Sole owner of the C allocations produced by one +/// `llama_rs_detect_reasoning_markers` call. Every pointer it holds is released +/// on drop, so no call path can leak or double-free them. +struct ReasoningMarkerDetection { status: llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers_status, - out_open: *const c_char, - out_close: *const c_char, + out_open: *mut c_char, + out_closes: *mut *mut c_char, + out_closes_count: usize, out_error: *mut c_char, -) -> Result<(Option, Option), MarkerDetectionError> { - match status { - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_OK => { - collect_optional_cstr_pair(out_open, out_close) +} + +impl ReasoningMarkerDetection { + fn detect(model: *const llama_cpp_bindings_sys::llama_model) -> Self { + let mut out_open: *mut c_char = ptr::null_mut(); + let mut out_closes: *mut *mut c_char = ptr::null_mut(); + let mut out_closes_count: usize = 0; + let mut out_error: *mut c_char = ptr::null_mut(); + + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers( + model, + &raw mut out_open, + &raw mut out_closes, + &raw mut out_closes_count, + &raw mut out_error, + ) + }; + + Self { + status, + out_open, + out_closes, + out_closes_count, + out_error, } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED => { - Err(MarkerDetectionError::NotEnoughMemory) + } + + fn read_closes(&self) -> Result, MarkerDetectionError> { + if self.out_closes.is_null() { + return Ok(Vec::new()); } - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_VENDORED_THREW_CXX_EXCEPTION => { - let message = unsafe { crate::ffi_error_reader::read_and_free_cpp_error(out_error) }; - Err(MarkerDetectionError::ReasoningMarkerDetectionFailed { message }) + + let entries = unsafe { std::slice::from_raw_parts(self.out_closes, self.out_closes_count) }; + + entries + .iter() + .filter_map(|entry| read_optional_owned_cstr(*entry).transpose()) + .collect() + } + + fn take_error_message(&mut self) -> String { + let message = unsafe { crate::ffi_error_reader::read_and_free_cpp_error(self.out_error) }; + self.out_error = ptr::null_mut(); + message + } + + fn into_result(mut self) -> Result { + match self.status { + llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_OK => { + Ok(DetectedReasoningMarkers { + open: read_optional_owned_cstr(self.out_open)?, + closes: self.read_closes()?, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED => { + Err(MarkerDetectionError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_VENDORED_THREW_CXX_EXCEPTION => { + Err(MarkerDetectionError::ReasoningMarkerDetectionFailed { + message: self.take_error_message(), + }) + } + other => unreachable!( + "llama_rs_detect_reasoning_markers returned unrecognized status {other}" + ), } - other => unreachable!( - "llama_rs_detect_reasoning_markers returned unrecognized status {other}" - ), + } +} + +impl Drop for ReasoningMarkerDetection { + fn drop(&mut self) { + unsafe { llama_cpp_bindings_sys::llama_rs_string_free(self.out_open) }; + unsafe { + llama_cpp_bindings_sys::llama_rs_string_array_free( + self.out_closes, + self.out_closes_count, + ); + }; + unsafe { llama_cpp_bindings_sys::llama_rs_string_free(self.out_error) }; } } @@ -1488,31 +1568,8 @@ const fn cxx_exception_owns_out_error( fn invoke_detect_reasoning_markers( model: *const llama_cpp_bindings_sys::llama_model, -) -> Result<(Option, Option), MarkerDetectionError> { - let mut out_open: *mut c_char = ptr::null_mut(); - let mut out_close: *mut c_char = ptr::null_mut(); - let mut out_error: *mut c_char = ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers( - model, - &raw mut out_open, - &raw mut out_close, - &raw mut out_error, - ) - }; - - let parsed = unsafe { - detect_reasoning_markers_status_to_result(status, out_open, out_close, out_error) - }; - - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_open) }; - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_close) }; - if !cxx_exception_owns_out_error(&parsed) { - unsafe { llama_cpp_bindings_sys::llama_rs_string_free(out_error) }; - } - - parsed +) -> Result { + ReasoningMarkerDetection::detect(model).into_result() } // SAFETY: `out_rendered` and `out_error` must be the pointers populated by the @@ -1592,7 +1649,7 @@ fn render_probe_messages( fn detect_reasoning_markers_via_template_probe( model: *const llama_cpp_bindings_sys::llama_model, -) -> Result, MarkerDetectionError> { +) -> Result, MarkerDetectionError> { use crate::extract_reasoning_markers_from_probe_renders::chunked_probe_messages_json; use crate::extract_reasoning_markers_from_probe_renders::extract_reasoning_markers_from_probe_renders; use crate::extract_reasoning_markers_from_probe_renders::plain_probe_messages_json; @@ -2106,11 +2163,12 @@ mod ffi_status_mapping_tests { use llama_cpp_bindings_types::ReasoningMarkers; use llama_cpp_bindings_types::ToolCallArguments; + use super::DetectedReasoningMarkers; + use super::ReasoningMarkerDetection; use super::ReasoningSplit; use super::chat_parser_create_status_to_result; use super::compute_tool_call_haystack_status_to_result; use super::cxx_exception_owns_out_error; - use super::detect_reasoning_markers_status_to_result; use super::diagnose_tool_call_synthetic_renders_status_to_result; use super::load_model_from_file_status_to_result; use super::outcome_from_via_ffi_result; @@ -2122,7 +2180,6 @@ mod ffi_status_mapping_tests { use super::parsed_chat_tool_call_count_status_to_result; use super::parsed_chat_tool_call_id_status_to_result; use super::parsed_chat_tool_call_name_status_to_result; - use super::reasoning_markers_from_marker_pair; use super::render_chat_template_status_to_result; use super::split_reasoning_prefix; use super::tokenize_status_to_result; @@ -2985,64 +3042,60 @@ mod ffi_status_mapping_tests { }; } + fn detection_with_status( + status: llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers_status, + ) -> ReasoningMarkerDetection { + ReasoningMarkerDetection { + status, + out_open: ptr::null_mut(), + out_closes: ptr::null_mut(), + out_closes_count: 0, + out_error: ptr::null_mut(), + } + } + #[test] - fn detect_reasoning_markers_ok_with_null_pointers_is_none_pair() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_OK, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; + fn detect_reasoning_markers_ok_with_null_pointers_yields_nothing_detected() { + let detected = + detection_with_status(llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_OK) + .into_result() + .expect("ok status must parse"); - assert_eq!(result, Ok((None, None))); + assert_eq!(detected.open, None); + assert!(detected.closes.is_empty()); } #[test] fn detect_reasoning_markers_allocation_failed_is_not_enough_memory() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; + let result = detection_with_status( + llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_ERROR_STRING_ALLOCATION_FAILED, + ) + .into_result(); - assert_eq!(result, Err(MarkerDetectionError::NotEnoughMemory)); + assert!(matches!(result, Err(MarkerDetectionError::NotEnoughMemory))); } #[test] fn detect_reasoning_markers_cxx_exception_is_detection_failed() { - let result = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_VENDORED_THREW_CXX_EXCEPTION, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; + let result = detection_with_status( + llama_cpp_bindings_sys::LLAMA_RS_DETECT_REASONING_MARKERS_VENDORED_THREW_CXX_EXCEPTION, + ) + .into_result(); - assert_eq!( + assert!(matches!( result, - Err(MarkerDetectionError::ReasoningMarkerDetectionFailed { - message: "unknown error".to_owned() - }) - ); + Err(MarkerDetectionError::ReasoningMarkerDetectionFailed { message }) + if message == "unknown error" + )); } #[test] #[should_panic(expected = "llama_rs_detect_reasoning_markers returned unrecognized status")] fn detect_reasoning_markers_unrecognized_status_panics() { - let _ = unsafe { - detect_reasoning_markers_status_to_result( - llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers_status::MAX, - ptr::null(), - ptr::null(), - ptr::null_mut(), - ) - }; + let _ = detection_with_status( + llama_cpp_bindings_sys::llama_rs_detect_reasoning_markers_status::MAX, + ) + .into_result(); } #[test] @@ -3329,7 +3382,7 @@ mod ffi_status_mapping_tests { fn split_reasoning_prefix_with_missing_open_marker_returns_content_only() { let markers = ReasoningMarkers { open: "".to_owned(), - close: "".to_owned(), + closes: vec!["".to_owned()], }; let ReasoningSplit { reasoning, content } = split_reasoning_prefix("plain answer", Some(&markers), ""); @@ -3342,7 +3395,7 @@ mod ffi_status_mapping_tests { fn split_reasoning_prefix_with_missing_close_marker_returns_content_only() { let markers = ReasoningMarkers { open: "".to_owned(), - close: "".to_owned(), + closes: vec!["".to_owned()], }; let ReasoningSplit { reasoning, content } = split_reasoning_prefix("unterminated", Some(&markers), ""); @@ -3355,7 +3408,7 @@ mod ffi_status_mapping_tests { fn split_reasoning_prefix_extracts_reasoning_and_trailing_content() { let markers = ReasoningMarkers { open: "".to_owned(), - close: "".to_owned(), + closes: vec!["".to_owned()], }; let ReasoningSplit { reasoning, content } = split_reasoning_prefix( "deduceanswertail", @@ -3368,32 +3421,52 @@ mod ffi_status_mapping_tests { } #[test] - fn reasoning_markers_from_marker_pair_with_both_present_builds_markers() { - let markers = reasoning_markers_from_marker_pair( - Some("".to_owned()), - Some("".to_owned()), - ); + fn detected_markers_with_both_present_build_markers() { + let markers = DetectedReasoningMarkers { + open: Some("".to_owned()), + closes: vec!["".to_owned()], + } + .into_reasoning_markers(); assert_eq!( markers, Some(ReasoningMarkers { open: "".to_owned(), - close: "".to_owned() + closes: vec!["".to_owned()], }) ); } #[test] - fn reasoning_markers_from_marker_pair_with_empty_marker_is_none() { - let markers = - reasoning_markers_from_marker_pair(Some(String::new()), Some("".to_owned())); + fn detected_markers_keep_every_close_candidate() { + let markers = DetectedReasoningMarkers { + open: "".to_owned().into(), + closes: vec!["".to_owned(), "".to_owned()], + } + .into_reasoning_markers() + .expect("both candidates are usable"); + + assert_eq!(markers.closes, vec!["", ""]); + } + + #[test] + fn detected_markers_with_only_empty_closes_are_none() { + let markers = DetectedReasoningMarkers { + open: Some("".to_owned()), + closes: vec![String::new()], + } + .into_reasoning_markers(); assert!(markers.is_none()); } #[test] - fn reasoning_markers_from_marker_pair_with_missing_marker_is_none() { - let markers = reasoning_markers_from_marker_pair(None, Some("".to_owned())); + fn detected_markers_with_missing_open_are_none() { + let markers = DetectedReasoningMarkers { + open: None, + closes: vec!["".to_owned()], + } + .into_reasoning_markers(); assert!(markers.is_none()); } diff --git a/llama-cpp-bindings/src/model/llama_load_mode_parse_error.rs b/llama-cpp-bindings/src/model/llama_load_mode_parse_error.rs new file mode 100644 index 000000000..811b8e14d --- /dev/null +++ b/llama-cpp-bindings/src/model/llama_load_mode_parse_error.rs @@ -0,0 +1,5 @@ +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LlamaLoadModeParseError { + pub value: llama_cpp_bindings_sys::llama_load_mode, + pub context: String, +} diff --git a/llama-cpp-bindings/src/model/load_mode.rs b/llama-cpp-bindings/src/model/load_mode.rs new file mode 100644 index 000000000..f02ffe200 --- /dev/null +++ b/llama-cpp-bindings/src/model/load_mode.rs @@ -0,0 +1,116 @@ +use crate::model::llama_load_mode_parse_error::LlamaLoadModeParseError; + +/// How llama.cpp should bring model weights into memory. +/// +/// [`LlamaLoadMode::Auto`] is upstream's default: it memory-maps the weights +/// unless a participating device reports no mmap support, in which case +/// llama.cpp falls back to a plain read. +#[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum LlamaLoadMode { + #[default] + Auto, + None, + Mmap, + Mlock, + MmapMlock, + DirectIo, +} + +impl LlamaLoadMode { + /// The same mode with memory mapping removed, keeping any mlock request. + /// + /// Upstream models the mode as one value rather than a bit set, so + /// dropping mmap is a remap rather than a bit clear. + #[must_use] + pub const fn without_mmap(self) -> Self { + match self { + Self::Auto | Self::None | Self::Mmap => Self::None, + Self::Mlock | Self::MmapMlock => Self::Mlock, + Self::DirectIo => Self::DirectIo, + } + } +} + +/// # Errors +/// Returns `LlamaLoadModeParseError` if the value does not correspond to a valid `LlamaLoadMode`. +impl TryFrom for LlamaLoadMode { + type Error = LlamaLoadModeParseError; + + fn try_from(value: llama_cpp_bindings_sys::llama_load_mode) -> Result { + match value { + llama_cpp_bindings_sys::LLAMA_LOAD_MODE_AUTO => Ok(Self::Auto), + llama_cpp_bindings_sys::LLAMA_LOAD_MODE_NONE => Ok(Self::None), + llama_cpp_bindings_sys::LLAMA_LOAD_MODE_MMAP => Ok(Self::Mmap), + llama_cpp_bindings_sys::LLAMA_LOAD_MODE_MLOCK => Ok(Self::Mlock), + llama_cpp_bindings_sys::LLAMA_LOAD_MODE_MMAP_MLOCK => Ok(Self::MmapMlock), + llama_cpp_bindings_sys::LLAMA_LOAD_MODE_DIRECT_IO => Ok(Self::DirectIo), + _ => Err(LlamaLoadModeParseError { + value, + context: format!("unknown load mode value: {value}"), + }), + } + } +} + +impl From for llama_cpp_bindings_sys::llama_load_mode { + fn from(value: LlamaLoadMode) -> Self { + match value { + LlamaLoadMode::Auto => llama_cpp_bindings_sys::LLAMA_LOAD_MODE_AUTO, + LlamaLoadMode::None => llama_cpp_bindings_sys::LLAMA_LOAD_MODE_NONE, + LlamaLoadMode::Mmap => llama_cpp_bindings_sys::LLAMA_LOAD_MODE_MMAP, + LlamaLoadMode::Mlock => llama_cpp_bindings_sys::LLAMA_LOAD_MODE_MLOCK, + LlamaLoadMode::MmapMlock => llama_cpp_bindings_sys::LLAMA_LOAD_MODE_MMAP_MLOCK, + LlamaLoadMode::DirectIo => llama_cpp_bindings_sys::LLAMA_LOAD_MODE_DIRECT_IO, + } + } +} + +#[cfg(test)] +mod tests { + use super::LlamaLoadMode; + + #[test] + fn try_from_invalid_reports_the_value() { + let result = LlamaLoadMode::try_from(99); + + assert!(result.is_err()); + assert_eq!(result.unwrap_err().value, 99); + } + + #[test] + fn every_variant_roundtrips_through_the_upstream_value() { + for mode in [ + LlamaLoadMode::Auto, + LlamaLoadMode::None, + LlamaLoadMode::Mmap, + LlamaLoadMode::Mlock, + LlamaLoadMode::MmapMlock, + LlamaLoadMode::DirectIo, + ] { + let raw = llama_cpp_bindings_sys::llama_load_mode::from(mode); + + assert_eq!(LlamaLoadMode::try_from(raw), Ok(mode)); + } + } + + #[test] + fn default_is_auto() { + assert_eq!(LlamaLoadMode::default(), LlamaLoadMode::Auto); + } + + #[test] + fn without_mmap_drops_mapping_and_keeps_mlock() { + assert_eq!(LlamaLoadMode::Auto.without_mmap(), LlamaLoadMode::None); + assert_eq!(LlamaLoadMode::None.without_mmap(), LlamaLoadMode::None); + assert_eq!(LlamaLoadMode::Mmap.without_mmap(), LlamaLoadMode::None); + assert_eq!(LlamaLoadMode::Mlock.without_mmap(), LlamaLoadMode::Mlock); + assert_eq!( + LlamaLoadMode::MmapMlock.without_mmap(), + LlamaLoadMode::Mlock + ); + assert_eq!( + LlamaLoadMode::DirectIo.without_mmap(), + LlamaLoadMode::DirectIo + ); + } +} diff --git a/llama-cpp-bindings/src/model/params.rs b/llama-cpp-bindings/src/model/params.rs index ebf864b04..d6391fa17 100644 --- a/llama-cpp-bindings/src/model/params.rs +++ b/llama-cpp-bindings/src/model/params.rs @@ -6,7 +6,9 @@ use std::ptr::null; use crate::LlamaCppError; use crate::context::params::LlamaContextParams; use crate::error::{FitError, ModelParamsError}; +use crate::model::llama_load_mode_parse_error::LlamaLoadModeParseError; use crate::model::llama_split_mode_parse_error::LlamaSplitModeParseError; +use crate::model::load_mode::LlamaLoadMode; use crate::model::params::fit_result::FitResult; use crate::model::params::kv_overrides::KvOverrides; use crate::model::split_mode::LlamaSplitMode; @@ -33,8 +35,7 @@ impl Debug for LlamaModelParams { .field("n_gpu_layers", &self.params.n_gpu_layers) .field("main_gpu", &self.params.main_gpu) .field("vocab_only", &self.params.vocab_only) - .field("use_mmap", &self.params.use_mmap) - .field("use_mlock", &self.params.use_mlock) + .field("load_mode", &self.load_mode()) .field("split_mode", &self.split_mode()) .field("devices", &self.devices) .field("kv_overrides", &"vec of kv_overrides") @@ -169,14 +170,10 @@ impl LlamaModelParams { self.params.vocab_only } - #[must_use] - pub const fn use_mmap(&self) -> bool { - self.params.use_mmap - } - - #[must_use] - pub const fn use_mlock(&self) -> bool { - self.params.use_mlock + /// # Errors + /// Returns `LlamaLoadModeParseError` if an unknown load mode is encountered. + pub fn load_mode(&self) -> Result { + LlamaLoadMode::try_from(self.params.load_mode) } /// # Errors @@ -229,8 +226,8 @@ impl LlamaModelParams { } #[must_use] - pub const fn with_use_mmap(mut self, use_mmap: bool) -> Self { - self.params.use_mmap = use_mmap; + pub fn with_load_mode(mut self, load_mode: LlamaLoadMode) -> Self { + self.params.load_mode = load_mode.into(); self } @@ -239,19 +236,17 @@ impl LlamaModelParams { self.params.no_alloc } - #[must_use] - pub const fn with_no_alloc(mut self, no_alloc: bool) -> Self { + /// Enabling `no_alloc` also drops memory mapping: weights that are never + /// read cannot be mapped. + /// + /// # Errors + /// Returns `LlamaLoadModeParseError` if an unknown load mode is encountered. + pub fn with_no_alloc(mut self, no_alloc: bool) -> Result { self.params.no_alloc = no_alloc; if no_alloc { - self.params.use_mmap = false; + self.params.load_mode = self.load_mode()?.without_mmap().into(); } - self - } - - #[must_use] - pub const fn with_use_mlock(mut self, use_mlock: bool) -> Self { - self.params.use_mlock = use_mlock; - self + Ok(self) } #[must_use] @@ -395,6 +390,7 @@ impl Default for LlamaModelParams { #[cfg(test)] mod tests { + use crate::model::load_mode::LlamaLoadMode; use crate::model::split_mode::LlamaSplitMode; use super::{LLAMA_CPP_MAX_DEVICES, LlamaModelParams}; @@ -417,8 +413,7 @@ mod tests { assert_eq!(params.n_gpu_layers(), -1); assert_eq!(params.main_gpu(), 0); assert!(!params.vocab_only()); - assert!(params.use_mmap()); - assert!(!params.use_mlock()); + assert_eq!(params.load_mode(), Ok(LlamaLoadMode::Auto)); assert_eq!(params.split_mode(), Ok(LlamaSplitMode::Layer)); assert!(params.devices().is_empty()); } @@ -473,62 +468,56 @@ mod tests { } #[test] - fn with_use_mmap_enables() { - let params = LlamaModelParams::default().with_use_mmap(true); + fn with_load_mode_sets_the_mode() { + let params = LlamaModelParams::default().with_load_mode(LlamaLoadMode::MmapMlock); - assert!(params.use_mmap()); - } - - #[test] - fn with_use_mmap_disables() { - let params = LlamaModelParams::default().with_use_mmap(false); - - assert!(!params.use_mmap()); + assert_eq!(params.load_mode(), Ok(LlamaLoadMode::MmapMlock)); } #[test] fn with_no_alloc_enables() { - let params = LlamaModelParams::default().with_no_alloc(true); + let params = LlamaModelParams::default() + .with_no_alloc(true) + .expect("default load mode is recognized"); assert!(params.no_alloc()); } #[test] fn with_no_alloc_disables() { - let params = LlamaModelParams::default().with_no_alloc(false); + let params = LlamaModelParams::default() + .with_no_alloc(false) + .expect("default load mode is recognized"); assert!(!params.no_alloc()); } #[test] - fn with_no_alloc_true_disables_mmap() { + fn with_no_alloc_true_drops_mapping_but_keeps_mlock() { let params = LlamaModelParams::default() - .with_use_mmap(true) - .with_no_alloc(true); + .with_load_mode(LlamaLoadMode::MmapMlock) + .with_no_alloc(true) + .expect("load mode is recognized"); assert!(params.no_alloc()); - assert!(!params.use_mmap()); + assert_eq!(params.load_mode(), Ok(LlamaLoadMode::Mlock)); } #[test] - fn default_no_alloc_is_false() { - let params = LlamaModelParams::default(); - - assert!(!params.no_alloc()); - } - - #[test] - fn with_use_mlock_enables() { - let params = LlamaModelParams::default().with_use_mlock(true); + fn with_no_alloc_false_leaves_the_load_mode_untouched() { + let params = LlamaModelParams::default() + .with_load_mode(LlamaLoadMode::Mmap) + .with_no_alloc(false) + .expect("load mode is recognized"); - assert!(params.use_mlock()); + assert_eq!(params.load_mode(), Ok(LlamaLoadMode::Mmap)); } #[test] - fn with_use_mlock_disables() { - let params = LlamaModelParams::default().with_use_mlock(false); + fn default_no_alloc_is_false() { + let params = LlamaModelParams::default(); - assert!(!params.use_mlock()); + assert!(!params.no_alloc()); } #[test] @@ -539,8 +528,7 @@ mod tests { assert!(debug_output.contains("n_gpu_layers")); assert!(debug_output.contains("main_gpu")); assert!(debug_output.contains("vocab_only")); - assert!(debug_output.contains("use_mmap")); - assert!(debug_output.contains("use_mlock")); + assert!(debug_output.contains("load_mode")); assert!(debug_output.contains("split_mode")); } @@ -551,13 +539,13 @@ mod tests { .with_main_gpu(1) .with_split_mode(LlamaSplitMode::Row) .with_vocab_only(true) - .with_use_mlock(true); + .with_load_mode(LlamaLoadMode::Mlock); assert_eq!(params.n_gpu_layers(), 10); assert_eq!(params.main_gpu(), 1); assert_eq!(params.split_mode(), Ok(LlamaSplitMode::Row)); assert!(params.vocab_only()); - assert!(params.use_mlock()); + assert_eq!(params.load_mode(), Ok(LlamaLoadMode::Mlock)); } #[test] diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index c3499e378..fd7a0ed8a 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -26,6 +26,13 @@ use crate::token::LlamaToken; pub use crate::ingest_outcome::IngestOutcome; pub use crate::sampled_token_section::SampledTokenSection; +#[derive(Clone, Copy, Debug)] +struct MatchedMarker { + span_start: usize, + length: usize, + kind: MarkerKind, +} + #[derive(Clone, Debug)] struct PendingToken { token: LlamaToken, @@ -174,25 +181,40 @@ impl<'model> SampledTokenClassifier<'model> { MarkerKind::ToolCallClose, ]; + let mut longest: Option = None; + for &kind in PROBE_KINDS { - let Some(marker) = self.markers.lookup(kind) else { - continue; - }; - if marker.is_empty() || self.pending.len() < marker.len() { - continue; - } - let span_start = self.pending.len() - marker.len(); - let matches = self - .pending - .iter() - .skip(span_start) - .zip(marker) - .all(|(entry, marker_token)| entry.token == *marker_token); - if matches { - self.mark_marker_span(span_start, kind); - return; + for marker in self.markers.lookup(kind) { + if marker.is_empty() || self.pending.len() < marker.len() { + continue; + } + let span_start = self.pending.len() - marker.len(); + let matches = self + .pending + .iter() + .skip(span_start) + .zip(marker) + .all(|(entry, marker_token)| entry.token == *marker_token); + if matches + && longest + .as_ref() + .is_none_or(|found| marker.len() > found.length) + { + longest = Some(MatchedMarker { + span_start, + length: marker.len(), + kind, + }); + } } } + + if let Some(MatchedMarker { + span_start, kind, .. + }) = longest + { + self.mark_marker_span(span_start, kind); + } } fn mark_marker_span(&mut self, span_start: usize, kind: MarkerKind) { @@ -548,10 +570,10 @@ mod tests { reasoning_close: Option>, ) -> StreamingMarkers { StreamingMarkers { - reasoning_open, - reasoning_close, - tool_call_open: None, - tool_call_close: None, + reasoning_open: reasoning_open.into_iter().collect(), + reasoning_close: reasoning_close.into_iter().collect(), + tool_call_open: Vec::new(), + tool_call_close: Vec::new(), } } @@ -621,6 +643,46 @@ mod tests { .collect() } + #[test] + fn the_longest_matching_close_candidate_wins_over_a_shorter_suffix() { + let markers = StreamingMarkers { + reasoning_open: vec![vec![token(100)]], + reasoning_close: vec![vec![token(202)], vec![token(201), token(202)]], + tool_call_open: Vec::new(), + tool_call_close: Vec::new(), + }; + let mut classifier = synthetic_classifier(markers); + classifier.section = SampledTokenSection::Reasoning; + + push_pending(&mut classifier, 201, ""); + classifier.try_consume_marker_at_tail(); + + assert!( + classifier.pending.iter().all(|entry| entry.is_boundary), + "the two-token candidate must claim both tokens, not just the trailing one" + ); + assert_eq!(classifier.section, SampledTokenSection::Content); + } + + #[test] + fn a_second_close_candidate_also_ends_the_reasoning_section() { + let markers = StreamingMarkers { + reasoning_open: vec![vec![token(100)]], + reasoning_close: vec![vec![token(200)], vec![token(300)]], + tool_call_open: Vec::new(), + tool_call_close: Vec::new(), + }; + let mut classifier = synthetic_classifier(markers); + classifier.section = SampledTokenSection::Reasoning; + + push_pending(&mut classifier, 300, ""); + classifier.try_consume_marker_at_tail(); + + assert_eq!(classifier.section, SampledTokenSection::Content); + } + #[test] fn single_token_close_marker_when_already_in_reasoning_emits_empty_piece_for_marker() { let markers = markers_with(Some(vec![token(100)]), Some(vec![token(200)])); @@ -745,10 +807,10 @@ mod tests { #[test] fn spurious_tool_call_close_in_reasoning_section_classifies_as_tool_call() { let markers = StreamingMarkers { - reasoning_open: Some(vec![token(100)]), - reasoning_close: Some(vec![token(200)]), - tool_call_open: Some(vec![token(300)]), - tool_call_close: Some(vec![token(400)]), + reasoning_open: vec![vec![token(100)]], + reasoning_close: vec![vec![token(200)]], + tool_call_open: vec![vec![token(300)]], + tool_call_close: vec![vec![token(400)]], }; let mut classifier = synthetic_classifier(markers); classifier.section = SampledTokenSection::ToolCall; @@ -1058,8 +1120,8 @@ mod tests { let returned = classifier.markers(); - assert_eq!(returned.reasoning_open.as_deref(), Some(&[token(1)][..])); - assert_eq!(returned.reasoning_close.as_deref(), Some(&[token(2)][..])); + assert_eq!(returned.reasoning_open, vec![vec![token(1)]]); + assert_eq!(returned.reasoning_close, vec![vec![token(2)]]); } #[test] @@ -1076,7 +1138,7 @@ mod tests { #[test] fn spurious_tool_call_close_in_content_section_classifies_as_content() { let mut markers = markers_with(None, None); - markers.tool_call_close = Some(vec![token(300)]); + markers.tool_call_close = vec![vec![token(300)]]; let mut classifier = synthetic_classifier(markers); classifier.section = SampledTokenSection::Content; @@ -1093,10 +1155,10 @@ mod tests { fn markers_with_tool_call_open(tool_call_open: Vec) -> StreamingMarkers { StreamingMarkers { - reasoning_open: None, - reasoning_close: None, - tool_call_open: Some(tool_call_open), - tool_call_close: None, + reasoning_open: Vec::new(), + reasoning_close: Vec::new(), + tool_call_open: vec![tool_call_open], + tool_call_close: Vec::new(), } } @@ -1303,10 +1365,10 @@ mod tests { #[test] fn json_probe_does_not_engage_in_reasoning_section() { let markers = StreamingMarkers { - reasoning_open: Some(vec![token(800)]), - reasoning_close: Some(vec![token(801)]), - tool_call_open: Some(vec![token(900)]), - tool_call_close: None, + reasoning_open: vec![vec![token(800)]], + reasoning_close: vec![vec![token(801)]], + tool_call_open: vec![vec![token(900)]], + tool_call_close: Vec::new(), }; let mut classifier = synthetic_classifier(markers); classifier.section = SampledTokenSection::Reasoning; diff --git a/llama-cpp-bindings/src/sampling.rs b/llama-cpp-bindings/src/sampling.rs index d2be11f78..ff70cfb88 100644 --- a/llama-cpp-bindings/src/sampling.rs +++ b/llama-cpp-bindings/src/sampling.rs @@ -125,18 +125,6 @@ fn sampler_init_grammar_lazy_patterns_status_to_result( } } -fn n_ctx_train_overflow_to_grammar_error(convert_error: std::num::TryFromIntError) -> GrammarError { - GrammarError::IntegerOverflow(format!( - "n_ctx_train does not fit into u32: {convert_error}" - )) -} - -fn checked_u32_as_i32(value: u32) -> Result { - i32::try_from(value).map_err(|convert_error| { - GrammarError::IntegerOverflow(format!("value exceeds i32::MAX: {convert_error}")) - }) -} - fn checked_usize_as_i32_sampling(value: usize) -> Result { i32::try_from(value).map_err(|convert_error| { SamplingError::IntegerOverflow(format!("value exceeds i32::MAX: {convert_error}")) @@ -498,14 +486,9 @@ impl LlamaSampler { .map(|seq_breaker| seq_breaker.as_ptr()) .collect(); - let n_ctx_train_value = model - .n_ctx_train() - .map_err(n_ctx_train_overflow_to_grammar_error)?; - let n_ctx_train = checked_u32_as_i32(n_ctx_train_value)?; let sampler = unsafe { llama_cpp_bindings_sys::llama_sampler_init_dry( model.vocab_ptr(), - n_ctx_train, multiplier, base, allowed_length, @@ -520,6 +503,7 @@ impl LlamaSampler { #[must_use] pub fn penalties( + n_vocab: i32, penalty_last_n: i32, penalty_repeat: f32, penalty_freq: f32, @@ -527,6 +511,7 @@ impl LlamaSampler { ) -> Self { let sampler = unsafe { llama_cpp_bindings_sys::llama_sampler_init_penalties( + n_vocab, penalty_last_n, penalty_repeat, penalty_freq, @@ -781,7 +766,7 @@ mod tests { #[test] fn accept_succeeds() { let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(32000, 64, 1.1, 0.0, 0.0), LlamaSampler::greedy(), ]); @@ -793,7 +778,7 @@ mod tests { #[test] fn try_accept_succeeds_on_penalties_sampler() { let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(32000, 64, 1.1, 0.0, 0.0), LlamaSampler::greedy(), ]); @@ -807,7 +792,7 @@ mod tests { use crate::token::LlamaToken; let mut sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(32000, 64, 1.1, 0.0, 0.0), LlamaSampler::greedy(), ]); @@ -821,7 +806,7 @@ mod tests { use crate::token::LlamaToken; let _sampler = LlamaSampler::chain_simple([ - LlamaSampler::penalties(64, 1.1, 0.0, 0.0), + LlamaSampler::penalties(32000, 64, 1.1, 0.0, 0.0), LlamaSampler::greedy(), ]) .with_tokens([LlamaToken::new(10), LlamaToken::new(20)]) @@ -863,12 +848,6 @@ mod tests { assert!(debug_output.contains("LlamaSampler")); } - #[test] - fn checked_u32_as_i32_overflow() { - let result = super::checked_u32_as_i32(u32::MAX); - assert!(result.is_err()); - } - #[test] fn checked_usize_as_i32_sampling_overflow() { let result = super::checked_usize_as_i32_sampling(usize::MAX); @@ -1105,17 +1084,6 @@ mod tests { ); } - #[test] - fn n_ctx_train_overflow_maps_to_integer_overflow() { - let convert_error = u32::try_from(-1_i64).expect_err("-1 cannot convert to u32"); - let grammar_error = super::n_ctx_train_overflow_to_grammar_error(convert_error); - - assert_eq!( - std::mem::discriminant(&grammar_error), - std::mem::discriminant(&GrammarError::IntegerOverflow(String::new())), - ); - } - #[test] fn grammar_returns_root_not_found_before_touching_model() { let model = unsafe { &*std::ptr::NonNull::::dangling().as_ptr() }; diff --git a/llama-cpp-bindings/src/streaming_markers.rs b/llama-cpp-bindings/src/streaming_markers.rs index 03be06b91..7781a88bb 100644 --- a/llama-cpp-bindings/src/streaming_markers.rs +++ b/llama-cpp-bindings/src/streaming_markers.rs @@ -1,51 +1,59 @@ use crate::marker_kind::MarkerKind; use crate::token::LlamaToken; +/// Token sequences that end a section, one list per [`MarkerKind`]. +/// +/// A kind carries several alternatives because upstream templates can close a +/// reasoning block with more than one tag — for example `` or a +/// `` that begins a tool call directly. An empty list means the kind +/// has no marker at all. #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct StreamingMarkers { - pub reasoning_open: Option>, - pub reasoning_close: Option>, - pub tool_call_open: Option>, - pub tool_call_close: Option>, + pub reasoning_open: Vec>, + pub reasoning_close: Vec>, + pub tool_call_open: Vec>, + pub tool_call_close: Vec>, } impl StreamingMarkers { #[must_use] pub const fn has_any(&self) -> bool { - self.reasoning_open.is_some() - || self.reasoning_close.is_some() - || self.tool_call_open.is_some() - || self.tool_call_close.is_some() + !self.reasoning_open.is_empty() + || !self.reasoning_close.is_empty() + || !self.tool_call_open.is_empty() + || !self.tool_call_close.is_empty() } #[must_use] pub fn max_token_len(&self) -> usize { [ - self.reasoning_open.as_deref(), - self.reasoning_close.as_deref(), - self.tool_call_open.as_deref(), - self.tool_call_close.as_deref(), + &self.reasoning_open, + &self.reasoning_close, + &self.tool_call_open, + &self.tool_call_close, ] .into_iter() .flatten() - .map(<[LlamaToken]>::len) + .map(Vec::len) .max() .unwrap_or(0) } #[must_use] - pub fn lookup(&self, kind: MarkerKind) -> Option<&[LlamaToken]> { + pub fn lookup(&self, kind: MarkerKind) -> &[Vec] { match kind { - MarkerKind::ReasoningOpen => self.reasoning_open.as_deref(), - MarkerKind::ReasoningClose => self.reasoning_close.as_deref(), - MarkerKind::ToolCallOpen => self.tool_call_open.as_deref(), - MarkerKind::ToolCallClose => self.tool_call_close.as_deref(), + MarkerKind::ReasoningOpen => &self.reasoning_open, + MarkerKind::ReasoningClose => &self.reasoning_close, + MarkerKind::ToolCallOpen => &self.tool_call_open, + MarkerKind::ToolCallClose => &self.tool_call_close, } } } #[cfg(test)] mod tests { + use crate::marker_kind::MarkerKind; + use super::StreamingMarkers; use crate::token::LlamaToken; @@ -63,11 +71,32 @@ mod tests { #[test] fn streaming_markers_max_token_len_takes_longest() { let markers = StreamingMarkers { - reasoning_open: Some(vec![token(1)]), - reasoning_close: Some(vec![token(2), token(3), token(4)]), - tool_call_open: Some(vec![token(5), token(6)]), - tool_call_close: None, + reasoning_open: vec![vec![token(1)]], + reasoning_close: vec![vec![token(2), token(3), token(4)]], + tool_call_open: vec![vec![token(5), token(6)]], + tool_call_close: Vec::new(), + }; + assert_eq!(markers.max_token_len(), 3); + } + + #[test] + fn max_token_len_spans_every_alternative_of_one_kind() { + let markers = StreamingMarkers { + reasoning_close: vec![vec![token(1)], vec![token(2), token(3), token(4)]], + ..StreamingMarkers::default() }; + assert_eq!(markers.max_token_len(), 3); } + + #[test] + fn lookup_returns_every_alternative_for_a_kind() { + let markers = StreamingMarkers { + reasoning_close: vec![vec![token(1)], vec![token(2)]], + ..StreamingMarkers::default() + }; + + assert_eq!(markers.lookup(MarkerKind::ReasoningClose).len(), 2); + assert!(markers.lookup(MarkerKind::ToolCallOpen).is_empty()); + } } diff --git a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs index 8e1f21a5c..be32c5b61 100644 --- a/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs +++ b/llama-cpp-bindings/src/tool_call_format/paired_quote_args.rs @@ -224,9 +224,22 @@ mod tests { use llama_cpp_bindings_types::ToolCallValueQuote; use serde_json::json; + use super::find_bare_value_end; use super::parse; use crate::error::PairedQuoteFailure; + #[test] + fn a_bare_value_ends_at_the_first_comma_close_marker_or_input_end() { + assert_eq!(find_bare_value_end("42, next", ""), 2); + assert_eq!(find_bare_value_end("42rest", ""), 2); + assert_eq!(find_bare_value_end("42", ""), 2); + assert_eq!( + find_bare_value_end("42", ""), + 2, + "an empty close marker must not terminate the scan" + ); + } + fn gemma4_markers() -> ToolCallMarkers { ToolCallMarkers { open: "<|tool_call>call:".to_owned(), diff --git a/llama-cpp-test-harness-macros/src/expand.rs b/llama-cpp-test-harness-macros/src/expand.rs index f4da0bd21..03d438ab4 100644 --- a/llama-cpp-test-harness-macros/src/expand.rs +++ b/llama-cpp-test-harness-macros/src/expand.rs @@ -10,6 +10,7 @@ use syn::parse::ParseStream; use syn::parse2; use crate::parsed_args::ParsedArgs; +use crate::parsed_load_mode::ParsedLoadMode; use crate::parsed_source::ParsedSource; struct StackedItems { @@ -105,6 +106,12 @@ fn build_mmproj_source_literal(source: Option<&ParsedSource>) -> TokenStream { } } +fn build_load_mode_literal(load_mode: ParsedLoadMode) -> TokenStream { + let variant = Ident::new(load_mode.variant_name(), Span::call_site()); + + quote! { ::llama_cpp_test_harness::LlamaLoadMode::#variant } +} + fn build_registration(args: &ParsedArgs, fn_name: &Ident) -> TokenStream { let trial_name = format!( "{fn_name}[{suffix}]", @@ -113,8 +120,7 @@ fn build_registration(args: &ParsedArgs, fn_name: &Ident) -> TokenStream { let model_source_literal = build_model_source_literal(&args.model_source); let mmproj_source_literal = build_mmproj_source_literal(args.mmproj_source.as_ref()); let gpu_layers = args.model_load_params.n_gpu_layers; - let use_mmap = args.model_load_params.use_mmap; - let use_mlock = args.model_load_params.use_mlock; + let load_mode_literal = build_load_mode_literal(args.model_load_params.load_mode); let context_size = args.context_params.n_ctx; let logical_batch = args.context_params.n_batch; let physical_batch = args.context_params.n_ubatch; @@ -135,8 +141,7 @@ fn build_registration(args: &ParsedArgs, fn_name: &Ident) -> TokenStream { mmproj_source: #mmproj_source_literal, model_load_params: ::llama_cpp_test_harness::ModelLoadParams { n_gpu_layers: #gpu_layers, - use_mmap: #use_mmap, - use_mlock: #use_mlock, + load_mode: #load_mode_literal, }, }, context_params: ::llama_cpp_test_harness::ContextParams { @@ -181,8 +186,7 @@ mod tests { quote! { model_source = HuggingFace("foo", "bar.gguf"), n_gpu_layers = 0, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 1, n_batch = 1, n_ubatch = 1 @@ -228,8 +232,7 @@ mod tests { let attribute = quote! { model_source = LocalPath("/abs/local.gguf"), n_gpu_layers = 0, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 1, n_batch = 1, n_ubatch = 1 @@ -253,8 +256,7 @@ mod tests { let attribute = quote! { model_source = HuggingFace("r", "f"), n_gpu_layers = 0, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 1, n_batch = 1, n_ubatch = 1, @@ -279,8 +281,7 @@ mod tests { let attribute = quote! { model_source = HuggingFace("r", "f"), n_gpu_layers = 0, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 1, n_batch = 1, n_ubatch = 1, @@ -402,8 +403,7 @@ mod tests { let second_attribute = quote! { model_source = HuggingFace("second", "second.gguf"), n_gpu_layers = 1, - use_mmap = false, - use_mlock = false, + load_mode = Mmap, n_ctx = 2, n_batch = 2, n_ubatch = 2 diff --git a/llama-cpp-test-harness-macros/src/lib.rs b/llama-cpp-test-harness-macros/src/lib.rs index 45a42cb60..bb1b246a5 100644 --- a/llama-cpp-test-harness-macros/src/lib.rs +++ b/llama-cpp-test-harness-macros/src/lib.rs @@ -6,6 +6,7 @@ mod expand; mod parsed_args; mod parsed_context_params; +mod parsed_load_mode; mod parsed_model_load_params; mod parsed_source; @@ -51,8 +52,7 @@ mod tests { let attribute = quote! { model_source = HuggingFace("r", "f"), n_gpu_layers = 0, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 1, n_batch = 1, n_ubatch = 1 diff --git a/llama-cpp-test-harness-macros/src/parsed_args.rs b/llama-cpp-test-harness-macros/src/parsed_args.rs index 74818b144..c3f251f95 100644 --- a/llama-cpp-test-harness-macros/src/parsed_args.rs +++ b/llama-cpp-test-harness-macros/src/parsed_args.rs @@ -12,14 +12,14 @@ use syn::parse::ParseStream; use syn::punctuated::Punctuated; use crate::parsed_context_params::ParsedContextParams; +use crate::parsed_load_mode::ParsedLoadMode; use crate::parsed_model_load_params::ParsedModelLoadParams; use crate::parsed_source::ParsedSource; const REQUIRED_FIELDS: &[&str] = &[ "model_source", "n_gpu_layers", - "use_mmap", - "use_mlock", + "load_mode", "n_ctx", "n_batch", "n_ubatch", @@ -98,8 +98,7 @@ struct AttributeAccumulator { model_source: Option, mmproj_source: Option, n_gpu_layers: Option, - use_mmap: Option, - use_mlock: Option, + load_mode: Option, n_ctx: Option, n_batch: Option, n_ubatch: Option, @@ -141,17 +140,8 @@ fn dispatch_field( "n_ubatch", )?); } - "use_mmap" => { - accumulator.use_mmap = Some(require_bool_lit( - literal_from_expression(value)?, - "use_mmap", - )?); - } - "use_mlock" => { - accumulator.use_mlock = Some(require_bool_lit( - literal_from_expression(value)?, - "use_mlock", - )?); + "load_mode" => { + accumulator.load_mode = Some(ParsedLoadMode::parse(value, "load_mode")?); } "embeddings" => { accumulator.embeddings = Some(require_bool_lit( @@ -185,6 +175,14 @@ fn dispatch_field( ), )); } + "use_mmap" | "use_mlock" => { + return Err(syn::Error::new_spanned( + identifier, + format!( + "field `{name}` was removed; upstream replaced both booleans with one value, so use `load_mode = Auto` (or None, Mmap, Mlock, MmapMlock, DirectIo)" + ), + )); + } other => { return Err(syn::Error::new_spanned( identifier, @@ -241,8 +239,7 @@ impl Parse for ParsedArgs { mmproj_source: accumulator.mmproj_source, model_load_params: ParsedModelLoadParams { n_gpu_layers: require(accumulator.n_gpu_layers, "n_gpu_layers", span)?, - use_mmap: require(accumulator.use_mmap, "use_mmap", span)?, - use_mlock: require(accumulator.use_mlock, "use_mlock", span)?, + load_mode: require(accumulator.load_mode, "load_mode", span)?, }, context_params: ParsedContextParams { n_ctx: require(accumulator.n_ctx, "n_ctx", span)?, @@ -262,13 +259,13 @@ mod tests { use syn::parse_str; use super::ParsedArgs; + use crate::parsed_load_mode::ParsedLoadMode; use crate::parsed_source::ParsedSource; const ALL_REQUIRED: &str = "\ model_source = HuggingFace(\"foo\", \"bar.gguf\"), \ n_gpu_layers = 0, \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 512, \ n_batch = 128, \ n_ubatch = 64"; @@ -289,8 +286,7 @@ mod tests { }, ); assert_eq!(parsed.model_load_params.n_gpu_layers, 0); - assert!(parsed.model_load_params.use_mmap); - assert!(!parsed.model_load_params.use_mlock); + assert_eq!(parsed.model_load_params.load_mode, ParsedLoadMode::Auto); assert_eq!(parsed.context_params.n_ctx, 512); assert_eq!(parsed.context_params.n_batch, 128); assert_eq!(parsed.context_params.n_ubatch, 64); @@ -302,8 +298,7 @@ mod tests { let source = "\ model_source = LocalPath(\"/abs/local/model.gguf\"), \ n_gpu_layers = 0, \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; @@ -343,8 +338,8 @@ mod tests { #[test] fn legacy_repo_field_is_rejected_with_migration_hint() { - let source = "repo = \"foo\", file = \"bar\", n_gpu_layers = 0, use_mmap = true, \ - use_mlock = false, n_ctx = 1, n_batch = 1, n_ubatch = 1"; + let source = "repo = \"foo\", file = \"bar\", n_gpu_layers = 0, load_mode = Auto, \ + n_ctx = 1, n_batch = 1, n_ubatch = 1"; let message = parse(source) .expect_err("legacy repo must be rejected") .to_string(); @@ -364,7 +359,7 @@ mod tests { #[test] fn missing_model_source_is_rejected() { - let source = "n_gpu_layers = 0, use_mmap = true, use_mlock = false, \ + let source = "n_gpu_layers = 0, load_mode = Auto, \ n_ctx = 1, n_batch = 1, n_ubatch = 1"; let message = parse(source) .expect_err("missing model_source must fail") @@ -378,8 +373,8 @@ mod tests { #[test] fn missing_n_ctx_is_rejected() { - let source = "model_source = HuggingFace(\"x\", \"y\"), n_gpu_layers = 0, use_mmap = true, \ - use_mlock = false, n_batch = 1, n_ubatch = 1"; + let source = "model_source = HuggingFace(\"x\", \"y\"), n_gpu_layers = 0, load_mode = Auto, \ + n_batch = 1, n_ubatch = 1"; let message = parse(source) .expect_err("missing n_ctx must fail") .to_string(); @@ -427,8 +422,7 @@ mod tests { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = some_const, \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; @@ -444,8 +438,7 @@ mod tests { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = \"nine\", \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; @@ -461,8 +454,8 @@ mod tests { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = 0, \ - use_mmap = 1, \ - use_mlock = false, \ + load_mode = Auto, \ + embeddings = 1, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; @@ -478,8 +471,7 @@ mod tests { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = 0, \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = -1, \ n_batch = 1, \ n_ubatch = 1"; @@ -496,8 +488,7 @@ mod tests { foo::bar = 1, \ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = 0, \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; @@ -513,8 +504,7 @@ mod tests { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = 0, \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 99999999999, \ n_batch = 1, \ n_ubatch = 1"; @@ -537,8 +527,7 @@ mod tests { fn missing_n_gpu_layers_is_rejected() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; @@ -553,41 +542,43 @@ mod tests { } #[test] - fn missing_use_mmap_is_rejected() { + fn missing_load_mode_is_rejected() { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = 0, \ - use_mlock = false, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; let message = parse(source) - .expect_err("missing use_mmap must fail") + .expect_err("missing load_mode must fail") .to_string(); assert!( - message.contains("missing required field `use_mmap`"), + message.contains("missing required field `load_mode`"), "got: {message}" ); } #[test] - fn missing_use_mlock_is_rejected() { - let source = "\ - model_source = HuggingFace(\"x\", \"y\"), \ - n_gpu_layers = 0, \ - use_mmap = true, \ - n_ctx = 1, \ - n_batch = 1, \ - n_ubatch = 1"; - let message = parse(source) - .expect_err("missing use_mlock must fail") + fn legacy_mmap_booleans_are_rejected_with_migration_hint() { + for field in ["use_mmap", "use_mlock"] { + let source = format!("{ALL_REQUIRED}, {field} = true"); + let message = parse(&source) + .expect_err("legacy boolean must be rejected") + .to_string(); + + assert!(message.contains("load_mode = Auto"), "{field}: {message}"); + } + } + + #[test] + fn load_mode_rejects_an_unknown_variant() { + let source = override_field("load_mode", "Mystery"); + let message = parse(&source) + .expect_err("unknown load mode must fail") .to_string(); - assert!( - message.contains("missing required field `use_mlock`"), - "got: {message}" - ); + assert!(message.contains("unknown load mode"), "got: {message}"); } #[test] @@ -595,8 +586,7 @@ mod tests { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = 0, \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 1, \ n_ubatch = 1"; let message = parse(source) @@ -614,8 +604,7 @@ mod tests { let source = "\ model_source = HuggingFace(\"x\", \"y\"), \ n_gpu_layers = 0, \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 1, \ n_batch = 1"; let message = parse(source) @@ -717,11 +706,10 @@ mod tests { } fn override_field(field: &str, replacement: &str) -> String { - let parts: [(&str, &str); 7] = [ + let parts: [(&str, &str); 6] = [ ("model_source", "HuggingFace(\"foo\", \"bar.gguf\")"), ("n_gpu_layers", "0"), - ("use_mmap", "true"), - ("use_mlock", "false"), + ("load_mode", "Auto"), ("n_ctx", "512"), ("n_batch", "128"), ("n_ubatch", "64"), @@ -752,8 +740,8 @@ mod tests { #[test] fn each_bool_dispatch_arm_rejects_non_literal_value() { - for field in ["use_mmap", "use_mlock"] { - let source = override_field(field, "some_const"); + for field in ["embeddings", "void_logs"] { + let source = append_field(field, "some_const"); let message = parse(&source).expect_err(field).to_string(); assert!(message.contains("literal"), "{field}: {message}"); @@ -772,8 +760,8 @@ mod tests { #[test] fn each_bool_dispatch_arm_rejects_wrong_literal_kind() { - for field in ["use_mmap", "use_mlock"] { - let source = override_field(field, "0"); + for field in ["embeddings", "void_logs"] { + let source = append_field(field, "0"); let message = parse(&source).expect_err(field).to_string(); assert!(message.contains("bool literal"), "{field}: {message}"); @@ -855,8 +843,7 @@ mod tests { let source = "\ model_source = Mystery(\"a\", \"b\"), \ n_gpu_layers = 0, \ - use_mmap = true, \ - use_mlock = false, \ + load_mode = Auto, \ n_ctx = 1, \ n_batch = 1, \ n_ubatch = 1"; diff --git a/llama-cpp-test-harness-macros/src/parsed_load_mode.rs b/llama-cpp-test-harness-macros/src/parsed_load_mode.rs new file mode 100644 index 000000000..dd323d147 --- /dev/null +++ b/llama-cpp-test-harness-macros/src/parsed_load_mode.rs @@ -0,0 +1,127 @@ +use syn::Expr; +use syn::ExprPath; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ParsedLoadMode { + Auto, + None, + Mmap, + Mlock, + MmapMlock, + DirectIo, +} + +impl ParsedLoadMode { + pub const fn variant_name(self) -> &'static str { + match self { + Self::Auto => "Auto", + Self::None => "None", + Self::Mmap => "Mmap", + Self::Mlock => "Mlock", + Self::MmapMlock => "MmapMlock", + Self::DirectIo => "DirectIo", + } + } + + pub fn parse(expression: &Expr, field: &str) -> syn::Result { + let Expr::Path(ExprPath { path, .. }) = expression else { + return Err(syn::Error::new_spanned( + expression, + format!( + "field `{field}` expects a bare variant name such as `Auto`, `Mmap`, or `MmapMlock`" + ), + )); + }; + let variant_ident = path.get_ident().ok_or_else(|| { + syn::Error::new_spanned( + path, + format!("field `{field}` expects the bare variant name, not a qualified path"), + ) + })?; + + match variant_ident.to_string().as_str() { + "Auto" => Ok(Self::Auto), + "None" => Ok(Self::None), + "Mmap" => Ok(Self::Mmap), + "Mlock" => Ok(Self::Mlock), + "MmapMlock" => Ok(Self::MmapMlock), + "DirectIo" => Ok(Self::DirectIo), + other => Err(syn::Error::new_spanned( + variant_ident, + format!( + "unknown load mode `{other}`; expected one of: Auto, None, Mmap, Mlock, MmapMlock, DirectIo" + ), + )), + } + } +} + +#[cfg(test)] +mod tests { + use syn::parse_str; + + use super::ParsedLoadMode; + + fn parse(source: &str) -> syn::Result { + let expression: syn::Expr = parse_str(source)?; + ParsedLoadMode::parse(&expression, "load_mode") + } + + #[test] + fn parses_every_known_variant() { + for (source, expected) in [ + ("Auto", ParsedLoadMode::Auto), + ("None", ParsedLoadMode::None), + ("Mmap", ParsedLoadMode::Mmap), + ("Mlock", ParsedLoadMode::Mlock), + ("MmapMlock", ParsedLoadMode::MmapMlock), + ("DirectIo", ParsedLoadMode::DirectIo), + ] { + assert_eq!(parse(source).expect("valid variant"), expected); + } + } + + #[test] + fn variant_name_matches_the_parsed_source() { + for source in ["Auto", "None", "Mmap", "Mlock", "MmapMlock", "DirectIo"] { + assert_eq!(parse(source).expect("valid variant").variant_name(), source); + } + } + + #[test] + fn unknown_variant_is_rejected() { + let message = parse("Mystery") + .expect_err("unknown variant must fail") + .to_string(); + + assert!(message.contains("unknown load mode"), "got: {message}"); + } + + #[test] + fn non_path_expression_is_rejected() { + let message = parse("\"Auto\"") + .expect_err("string literal must fail") + .to_string(); + + assert!(message.contains("bare variant name"), "got: {message}"); + } + + #[test] + fn qualified_path_variant_is_rejected() { + let message = parse("LlamaLoadMode::Auto") + .expect_err("qualified path must fail") + .to_string(); + + assert!(message.contains("not a qualified path"), "got: {message}"); + } + + #[test] + fn unparseable_input_returns_err() { + let result = parse("@&^!"); + + assert!( + result.is_err(), + "garbage input must fail to parse as syn::Expr" + ); + } +} diff --git a/llama-cpp-test-harness-macros/src/parsed_model_load_params.rs b/llama-cpp-test-harness-macros/src/parsed_model_load_params.rs index 07c5368cb..56e2ea9da 100644 --- a/llama-cpp-test-harness-macros/src/parsed_model_load_params.rs +++ b/llama-cpp-test-harness-macros/src/parsed_model_load_params.rs @@ -1,6 +1,7 @@ +use crate::parsed_load_mode::ParsedLoadMode; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ParsedModelLoadParams { pub n_gpu_layers: i32, - pub use_mmap: bool, - pub use_mlock: bool, + pub load_mode: ParsedLoadMode, } diff --git a/llama-cpp-test-harness/src/execution_phase.rs b/llama-cpp-test-harness/src/execution_phase.rs index 6892889d8..d28a02507 100644 --- a/llama-cpp-test-harness/src/execution_phase.rs +++ b/llama-cpp-test-harness/src/execution_phase.rs @@ -83,6 +83,8 @@ impl ExecutionPhase { #[cfg(test)] mod tests { + use llama_cpp_bindings::model::load_mode::LlamaLoadMode; + use crate::ModelSource; use crate::load_key::LoadKey; use crate::model_load_params::ModelLoadParams; @@ -96,8 +98,7 @@ mod tests { mmproj_source: None, model_load_params: ModelLoadParams { n_gpu_layers: 7, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, }, registrations: Vec::new(), diff --git a/llama-cpp-test-harness/src/execution_plan.rs b/llama-cpp-test-harness/src/execution_plan.rs index 657fddeef..189b3188c 100644 --- a/llama-cpp-test-harness/src/execution_plan.rs +++ b/llama-cpp-test-harness/src/execution_plan.rs @@ -63,6 +63,8 @@ impl ExecutionPlan { #[cfg(test)] mod tests { + use llama_cpp_bindings::model::load_mode::LlamaLoadMode; + use crate::context_params::ContextParams; use crate::llama_test_registration::LlamaTestRegistration; use crate::load_key::LoadKey; @@ -100,8 +102,7 @@ mod tests { mmproj_source: None, model_load_params: ModelLoadParams { n_gpu_layers: 0, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, }, context_params: TRIVIAL_CONTEXT_PARAMS, @@ -118,8 +119,7 @@ mod tests { mmproj_source: None, model_load_params: ModelLoadParams { n_gpu_layers: 0, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, }, context_params: TRIVIAL_CONTEXT_PARAMS, @@ -136,8 +136,7 @@ mod tests { mmproj_source: None, model_load_params: ModelLoadParams { n_gpu_layers: 0, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, }, context_params: TRIVIAL_CONTEXT_PARAMS, @@ -154,8 +153,7 @@ mod tests { mmproj_source: None, model_load_params: ModelLoadParams { n_gpu_layers: 0, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, }, context_params: ALTERNATE_CONTEXT_PARAMS, @@ -173,8 +171,7 @@ mod tests { mmproj_source: None, model_load_params: ModelLoadParams { n_gpu_layers: 0, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, }, context_params: TRIVIAL_CONTEXT_PARAMS, diff --git a/llama-cpp-test-harness/src/lib.rs b/llama-cpp-test-harness/src/lib.rs index bcdeec72d..a6accf97a 100644 --- a/llama-cpp-test-harness/src/lib.rs +++ b/llama-cpp-test-harness/src/lib.rs @@ -39,6 +39,7 @@ pub use crate::no_op::no_op; pub use crate::phase_state::PhaseState; pub use crate::run::run; pub use crate::run_to_conclusions::run_to_conclusions; +pub use llama_cpp_bindings::model::load_mode::LlamaLoadMode; pub use llama_cpp_test_harness_macros::llama_test; #[doc(hidden)] diff --git a/llama-cpp-test-harness/src/load_key.rs b/llama-cpp-test-harness/src/load_key.rs index 5fad7200c..cbb5d58d6 100644 --- a/llama-cpp-test-harness/src/load_key.rs +++ b/llama-cpp-test-harness/src/load_key.rs @@ -53,6 +53,8 @@ impl LoadKey { #[cfg(test)] mod tests { + use llama_cpp_bindings::model::load_mode::LlamaLoadMode; + use crate::mmproj_source::MmprojSource; use crate::model_load_params::ModelLoadParams; use crate::model_source::ModelSource; @@ -68,8 +70,7 @@ mod tests { mmproj_source: None, model_load_params: ModelLoadParams { n_gpu_layers: 0, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, } } @@ -138,8 +139,7 @@ mod tests { mmproj_source: None, model_load_params: ModelLoadParams { n_gpu_layers: 0, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, }; @@ -165,8 +165,7 @@ mod tests { }), model_load_params: ModelLoadParams { n_gpu_layers: 0, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, }; @@ -190,8 +189,7 @@ mod tests { mmproj_source: Some(MmprojSource::LocalPath(NON_GGUF_PATH)), model_load_params: ModelLoadParams { n_gpu_layers: 0, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }, }; diff --git a/llama-cpp-test-harness/src/model_load_params.rs b/llama-cpp-test-harness/src/model_load_params.rs index 361e88c36..34098f391 100644 --- a/llama-cpp-test-harness/src/model_load_params.rs +++ b/llama-cpp-test-harness/src/model_load_params.rs @@ -1,10 +1,10 @@ +use llama_cpp_bindings::model::load_mode::LlamaLoadMode; use llama_cpp_bindings::model::params::LlamaModelParams; #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct ModelLoadParams { pub n_gpu_layers: i32, - pub use_mmap: bool, - pub use_mlock: bool, + pub load_mode: LlamaLoadMode, } impl ModelLoadParams { @@ -12,45 +12,41 @@ impl ModelLoadParams { pub fn into_llama_model_params(self) -> LlamaModelParams { let Self { n_gpu_layers, - use_mmap, - use_mlock, + load_mode, } = self; LlamaModelParams::default() .with_n_gpu_layers(n_gpu_layers) - .with_use_mmap(use_mmap) - .with_use_mlock(use_mlock) + .with_load_mode(load_mode) } } #[cfg(test)] mod tests { + use llama_cpp_bindings::model::load_mode::LlamaLoadMode; + use super::ModelLoadParams; #[test] - fn into_llama_model_params_carries_all_three_fields() { + fn into_llama_model_params_carries_both_fields() { let params = ModelLoadParams { n_gpu_layers: 7, - use_mmap: false, - use_mlock: true, + load_mode: LlamaLoadMode::Mlock, } .into_llama_model_params(); assert_eq!(params.n_gpu_layers(), 7); - assert!(!params.use_mmap()); - assert!(params.use_mlock()); + assert_eq!(params.load_mode(), Ok(LlamaLoadMode::Mlock)); } #[test] fn identical_values_compare_equal() { let one = ModelLoadParams { n_gpu_layers: 1, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }; let two = ModelLoadParams { n_gpu_layers: 1, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }; assert_eq!(one, two); @@ -60,13 +56,25 @@ mod tests { fn differing_n_gpu_layers_compare_unequal() { let one = ModelLoadParams { n_gpu_layers: 1, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, }; let two = ModelLoadParams { n_gpu_layers: 2, - use_mmap: true, - use_mlock: false, + load_mode: LlamaLoadMode::Auto, + }; + + assert_ne!(one, two); + } + + #[test] + fn differing_load_mode_compares_unequal() { + let one = ModelLoadParams { + n_gpu_layers: 1, + load_mode: LlamaLoadMode::Auto, + }; + let two = ModelLoadParams { + n_gpu_layers: 1, + load_mode: LlamaLoadMode::Mmap, }; assert_ne!(one, two); diff --git a/llama-cpp-test-harness/tests/harness_self_test.rs b/llama-cpp-test-harness/tests/harness_self_test.rs index 333e0f4af..2f9e004b2 100644 --- a/llama-cpp-test-harness/tests/harness_self_test.rs +++ b/llama-cpp-test-harness/tests/harness_self_test.rs @@ -10,8 +10,7 @@ use llama_cpp_test_harness::run_to_conclusions; #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -26,8 +25,7 @@ fn phase_a_first_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64 @@ -40,8 +38,7 @@ fn phase_a_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64 @@ -53,8 +50,7 @@ fn phase_a_intentionally_failing_trial(_fixture: &LlamaFixture<'_>) -> Result<() #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64 @@ -67,8 +63,7 @@ fn phase_b_first_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("Qwen/Qwen3-Embedding-0.6B-GGUF", "Qwen3-Embedding-0.6B-Q8_0.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64 @@ -83,8 +78,7 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64 @@ -92,8 +86,7 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("intentee-test-harness/does-not-exist", "no-such-file.gguf"), n_gpu_layers = 0, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 1, n_batch = 1, n_ubatch = 1 @@ -101,8 +94,7 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -111,8 +103,7 @@ fn phase_b_second_passing_trial(fixture: &LlamaFixture<'_>) -> Result<()> { #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64, @@ -126,8 +117,7 @@ fn shared_setup_failure_and_phase_a_trial(fixture: &LlamaFixture<'_>) -> Result< #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, - use_mmap = true, - use_mlock = false, + load_mode = Auto, n_ctx = 512, n_batch = 128, n_ubatch = 64,