diff --git a/Cargo.lock b/Cargo.lock index e53021fc3..5d7b2e208 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,6 +74,17 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "annotate-snippets" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" +dependencies = [ + "anstyle", + "memchr", + "unicode-width 0.2.2", +] + [[package]] name = "anstream" version = "1.0.0" @@ -130,6 +141,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayvec" version = "0.7.8" @@ -1148,6 +1165,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1454,6 +1480,16 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "granit-parser" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f99e46474f56bd905c56e817ebddcf377a611f94c53ac4649e4d3fa3c0cd0" +dependencies = [ + "arraydeque", + "smallvec", +] + [[package]] name = "h2" version = "0.4.18" @@ -2274,8 +2310,8 @@ dependencies = [ "safetensors 0.7.0", "sentencepiece", "serde", + "serde-saphyr", "serde_json", - "serde_yaml", "sha2 0.10.9", "socket2", "symphonia", @@ -2334,8 +2370,8 @@ dependencies = [ "mlxcel-core", "safetensors 0.7.0", "serde", + "serde-saphyr", "serde_json", - "serde_yaml", "tempfile", "thiserror 2.0.21", ] @@ -3569,6 +3605,20 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-saphyr" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8050abb251097357e24aff63ba2c52a6309ecb7d23a5474023df960a02694d8" +dependencies = [ + "annotate-snippets", + "encoding_rs_io", + "granit-parser", + "num-traits", + "serde_core", + "smallvec", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -3635,19 +3685,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "sha1" version = "0.10.7" @@ -4584,12 +4621,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index e53559eef..23cc78ba4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -166,7 +166,7 @@ mlxcel-xla = { path = "src/lib/mlxcel-xla", optional = true } # Serialization serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["preserve_order"] } -serde_yaml = "0.9" +serde-saphyr = { version = "1.3", default-features = false, features = ["deserialize"] } # Tensors & Models safetensors = "0.7.0" diff --git a/TECHNICAL_REPORTS/2279-maintained-yaml-parser-20261012.en.md b/TECHNICAL_REPORTS/2279-maintained-yaml-parser-20261012.en.md new file mode 100644 index 000000000..9e1cbe0ee --- /dev/null +++ b/TECHNICAL_REPORTS/2279-maintained-yaml-parser-20261012.en.md @@ -0,0 +1,33 @@ +# Technical report: PR #2279 — maintained YAML parser + +**Date**: 2026-10-12 +**Status**: Verified +**Risk level**: Medium + +## Executive summary + +The CLI and surgery configuration loaders now use deserialize-only serde-saphyr 1.3.0. This removes serde_yaml and unsafe-libyaml, the dependency chain that blocked CI under RUSTSEC-2024-0449. + +## Problem statement + +Both configuration paths depended on the unmaintained parser. An advisory suppression would leave the unsupported dependency in the runtime graph. The replacement must preserve document-order language priorities, schema validation, and operation-specific numeric errors. + +## Technical decisions + +Both loaders retain ordinary YAML merge keys and YAML 1.2 boolean interpretation. Duplicate mapping keys remain errors; a supported language key's duplicate failure maps to DuplicateLanguageCode. The streaming schema rejects unknown fields before they can be misclassified as language keys. + +Serde's untagged and internally tagged buffering reads numbers through a typeless representation. serde-saphyr represents allowed non-finite typeless values as canonical strings, so the loaders opt into that representation and narrowly recover .nan, .inf, and -.inf. Surgery's existing validators still reject non-finite factors, coefficients, and ratios. Ordinary quoted surgery numbers remain invalid. Deliberate compatibility change: quoted canonical special values now use the same handling as unquoted values; parser diagnostic formatting also changes. + +## Change summary + +The lockfile removes both unsupported packages and adds only the replacement parser's dependency tree. Regression tests cover duplicate schema fields, ordered numeric entries, boolean spellings, non-finite operation validation, and all six shipped surgery YAML examples. Existing public signatures and model execution paths are unchanged. + +## Validation + +All 34 focused language-bias tests and 28 surgery configuration tests pass, including the added factor/alpha/ratio non-finite and quoted canonical regressions. cargo deny check passes advisories, bans, licenses, and sources. Formatting and diff checks pass. Repository contract checks pass, including 123 Python tooling tests and 49 WebUI fixtures. Final tests and CI Clippy, CPU-only link, and cargo-deny were verified on dd36abb9b321fa6eb6c9abaca07f5690ead72a17. The duplicate local CUDA Clippy rebuild was stopped after CI Clippy succeeded. + +## References + +- Issue #2278 +- https://rustsec.org/advisories/RUSTSEC-2024-0449.html +- https://docs.rs/serde-saphyr/1.3.0/serde_saphyr/ diff --git a/TECHNICAL_REPORTS/2279-maintained-yaml-parser-20261012.ko.md b/TECHNICAL_REPORTS/2279-maintained-yaml-parser-20261012.ko.md new file mode 100644 index 000000000..fcf4a804d --- /dev/null +++ b/TECHNICAL_REPORTS/2279-maintained-yaml-parser-20261012.ko.md @@ -0,0 +1,33 @@ +# 기술 보고서: PR #2279 — 유지보수되는 YAML 파서 + +**작성일**: 2026-10-12 +**상태**: 검증 완료 +**위험도**: 중간 + +## 요약 + +CLI와 surgery 설정 로더가 역직렬화 기능만 활성화한 serde-saphyr 1.3.0을 사용합니다. RUSTSEC-2024-0449로 CI를 차단했던 serde_yaml과 unsafe-libyaml 의존성 경로를 제거합니다. + +## 문제 정의 + +두 설정 경로가 유지보수되지 않는 파서에 의존했습니다. 보안 권고를 무시하면 지원되지 않는 의존성이 실행 경로에 남습니다. 교체 과정에서 문서 순서에 따른 언어 우선순위, 스키마 검증, 작업별 숫자 오류를 보존해야 합니다. + +## 기술적 선택과 그 이유 + +두 로더는 YAML 병합 키를 일반 키로 처리하며 YAML 1.2 불리언 해석을 유지합니다. 중복 매핑 키는 계속 오류이며, 지원되는 언어 키의 중복 오류는 DuplicateLanguageCode로 변환합니다. 스트리밍 스키마 검증이 알 수 없는 필드를 먼저 거부하므로 이를 언어 키로 잘못 분류하지 않습니다. + +Serde의 untagged 및 internally tagged 버퍼링은 숫자를 타입이 정해지지 않은 표현으로 읽습니다. serde-saphyr는 허용된 비유한 숫자를 정규화된 문자열로 표현하므로 로더에서 이 표현을 활성화하고 .nan, .inf, -.inf만 복원합니다. 기존 surgery 검증은 비유한 배율, 계수, 비율을 계속 거부합니다. 일반적인 따옴표 숫자는 계속 유효하지 않습니다. 명시적인 호환성 변경은 따옴표로 감싼 정규화된 특수 값도 따옴표 없는 값과 같은 처리를 받는 점과 파서 진단 출력 형식입니다. + +## 변경 요약 + +잠금 파일에서 지원되지 않는 두 패키지를 제거하고 대체 파서의 의존성만 추가했습니다. 회귀 테스트는 중복 스키마 필드, 숫자 항목 순서, 불리언 표기, 비유한 숫자의 작업 검증 및 배포된 surgery YAML 예제 6개를 다룹니다. 공개 함수 시그니처와 모델 실행 경로는 그대로입니다. + +## 검증 + +언어 바이어스 집중 테스트 34개와 surgery 설정 테스트 28개를 통과했습니다. 추가한 factor/alpha/ratio 비유한 숫자 및 따옴표 특수 값 회귀 테스트도 통과했습니다. cargo deny check의 advisories, bans, licenses, sources 검사와 포맷 및 패치 검사를 통과했습니다. Python 도구 테스트 123개와 WebUI 픽스처 49개를 포함한 저장소 계약 검사도 통과했습니다. 최종 테스트와 CI Clippy, CPU 전용 링크, cargo-deny는 dd36abb9b321fa6eb6c9abaca07f5690ead72a17에서 확인했습니다. 중복된 로컬 CUDA Clippy 재빌드는 CI Clippy 성공 후 중단했습니다. + +## 참고 자료 + +- 이슈 #2278 +- https://rustsec.org/advisories/RUSTSEC-2024-0449.html +- https://docs.rs/serde-saphyr/1.3.0/serde_saphyr/ diff --git a/src/lang_bias.rs b/src/lang_bias.rs index 113ef06e6..392a0c57b 100644 --- a/src/lang_bias.rs +++ b/src/lang_bias.rs @@ -210,10 +210,10 @@ pub struct LangBiasYamlConfig { /// which made the resolve loop's duplicate check unreachable and let the YAML /// path silently accept input the `--lang-bias` parser rejects. See issue #1267. /// -/// Collecting the entries through `MapAccess` into a `Vec` keeps both: the -/// author's order survives, and a repeated key arrives as a second entry that -/// the resolve loop can reject. The accepted YAML syntax is unchanged, `bias:` -/// is still a plain mapping. +/// Collecting entries through `MapAccess` into a `Vec` preserves their priority +/// order. The YAML parser rejects duplicate mapping keys before this visitor +/// receives them; `load_yaml_config` retains the specific language-code error. +/// The accepted YAML syntax is unchanged: `bias:` is still a plain mapping. #[derive(Debug, Default)] pub struct BiasEntries(Vec<(String, BiasValueStr)>); @@ -223,7 +223,7 @@ impl BiasEntries { &self.0 } - /// Number of entries, counting a repeated language code once per occurrence. + /// Number of entries in document order. pub fn len(&self) -> usize { self.0.len() } @@ -262,8 +262,8 @@ impl<'de> Deserialize<'de> for BiasEntries { M: serde::de::MapAccess<'de>, { let mut entries = Vec::with_capacity(access.size_hint().unwrap_or(0)); - // Deliberately a Vec push per entry rather than a map insert: - // repeated keys must reach the caller so it can reject them. + // Preserve parser-provided document order with a Vec push. + // The loader handles parser-level duplicate-key errors. while let Some(entry) = access.next_entry::()? { entries.push(entry); } @@ -313,7 +313,12 @@ impl<'de> Deserialize<'de> for BiasValueStr { let raw = RawBias::deserialize(deserializer)?; let value = match raw { RawBias::Float(f) => f as f32, - RawBias::Str(s) => parse_bias_f32(&s).map_err(serde::de::Error::custom)?, + RawBias::Str(s) => match s.as_str() { + ".nan" => f32::NAN, + ".inf" => f32::INFINITY, + "-.inf" => f32::NEG_INFINITY, + _ => parse_bias_f32(&s).map_err(serde::de::Error::custom)?, + }, }; Ok(BiasValueStr(value)) } @@ -355,9 +360,25 @@ pub fn load_yaml_config(path: &PathBuf) -> Result source: e, })?; - serde_yaml::from_str(&content).map_err(|e| CliError::ConfigParseError { - path: path.display().to_string(), - message: e.to_string(), + let options = serde_saphyr::options! { + merge_keys: serde_saphyr::MergeKeyPolicy::AsOrdinary, + strict_booleans: true, + reject_non_finite_typeless_float: false, + }; + serde_saphyr::from_str_with_options(&content, options).map_err(|e| { + // Only bias mapping keys can be valid language codes in this schema. + // Keep the CLI diagnostic while retaining strict parser duplicate checks. + if let serde_saphyr::Error::DuplicateMappingKey { + key: Some(code), .. + } = e.without_snippet() + && LanguageCode::from_str(code).is_ok() + { + return CliError::DuplicateLanguageCode { code: code.clone() }; + } + CliError::ConfigParseError { + path: path.display().to_string(), + message: e.to_string(), + } }) } @@ -495,10 +516,8 @@ impl LangBiasCliArgs { } if let Some(yaml_bias) = yaml.bias { - // `BiasEntries` yields the `bias:` block in document order and - // keeps repeated keys, so `ordered` ends up in the priority - // order the author wrote and the duplicate check below actually - // fires. This mirrors `parse_lang_bias_entries` entry for entry. + // Preserve the author's priority order. The YAML loader rejects + // duplicates; retain this defensive check for other deserializers. let mut ordered = Vec::new(); let mut seen: HashMap = HashMap::new(); for (code_str, BiasValueStr(bias)) in yaml_bias { @@ -692,7 +711,7 @@ exceptions: include_numeric: false include_punctuation: true "#; - let config: LangBiasYamlConfig = serde_yaml::from_str(yaml_str).unwrap(); + let config: LangBiasYamlConfig = serde_saphyr::from_str(yaml_str).unwrap(); assert!(matches!(config.policy, Some(PolicyStr::Conservative))); let bias = config.bias.unwrap(); // Assert the order, not just membership. Asserting membership alone is @@ -714,7 +733,7 @@ exceptions: let yaml_str = r#" policy: strict "#; - let config: LangBiasYamlConfig = serde_yaml::from_str(yaml_str).unwrap(); + let config: LangBiasYamlConfig = serde_saphyr::from_str(yaml_str).unwrap(); assert!(config.bias.is_none()); } @@ -724,7 +743,7 @@ policy: strict policy: conservative unknown_field: value "#; - let result: Result = serde_yaml::from_str(yaml_str); + let result: Result = serde_saphyr::from_str(yaml_str); assert!( result.is_err(), "unknown top-level key should produce a parse error" @@ -904,11 +923,8 @@ unknown_field: value #[test] fn yaml_duplicate_language_code_is_rejected() { - // serde_yaml deserializes a repeated key into a typed `HashMap` without - // any diagnostic (last occurrence wins), which is why the duplicate - // check in `resolve` used to be unreachable. The ordered representation - // delivers both occurrences, so the YAML path now rejects what the - // `--lang-bias` path already rejected. + // The parser rejects duplicates before collecting bias entries. The + // loader maps that failure to the same specific error as the CLI path. let (_tmpfile, path) = write_temp_yaml("bias:\n ja: -1.0\n zh: -2.0\n ja: -3.0\n"); let args = LangBiasCliArgs { @@ -945,7 +961,7 @@ unknown_field: value // The accepted syntax is unchanged: `bias:` is a mapping, and a list // form is still a parse error rather than a silently different config. let result: Result = - serde_yaml::from_str("bias:\n - ja: -inf\n - zh: -10.0\n"); + serde_saphyr::from_str("bias:\n - ja: -inf\n - zh: -10.0\n"); assert!( result.is_err(), "a sequence-shaped bias: block should still be a parse error" @@ -1164,3 +1180,7 @@ unknown_field: value assert_eq!(parse_bias_f32("-5.0").unwrap(), -5.0_f32); } } + +#[cfg(test)] +#[path = "lang_bias_yaml_compat_tests.rs"] +mod yaml_compat_tests; diff --git a/src/lang_bias_yaml_compat_tests.rs b/src/lang_bias_yaml_compat_tests.rs new file mode 100644 index 000000000..61404b0b0 --- /dev/null +++ b/src/lang_bias_yaml_compat_tests.rs @@ -0,0 +1,78 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +fn load(text: &str) -> Result { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(file.path(), text).unwrap(); + load_yaml_config(&file.path().to_path_buf()) +} + +#[test] +fn duplicate_schema_fields_remain_parse_errors() { + for text in [ + "policy: strict\npolicy: conservative\n", + "en: 1\nen: 2\n", + "exceptions: {en: 1, en: 2}\n", + "bias: {ja: 1}\nbias: {ko: 2}\n", + "exceptions: {include_special: true, include_special: false}\n", + ] { + assert!(matches!(load(text), Err(CliError::ConfigParseError { .. }))); + } +} + +#[test] +fn numeric_spellings_and_order_are_preserved() { + let config = + load("bias: {ja: -inf, zh: +inf, ko: .nan, en: 0x10, ru: 1e2, ar: '2.5'}").unwrap(); + let entries = config.bias.unwrap().0; + assert_eq!( + entries + .iter() + .map(|(code, _)| code.as_str()) + .collect::>(), + ["ja", "zh", "ko", "en", "ru", "ar"] + ); + assert_eq!(entries[0].1.0, f32::NEG_INFINITY); + assert_eq!(entries[1].1.0, f32::INFINITY); + assert!(entries[2].1.0.is_nan()); + assert_eq!(entries[3].1.0, 16.0); + assert_eq!(entries[4].1.0, 100.0); + assert_eq!(entries[5].1.0, 2.5); +} + +#[test] +fn yaml_merge_does_not_bypass_schema_validation() { + assert!(matches!( + load("<<: {policy: strict}\n"), + Err(CliError::ConfigParseError { .. }) + )); +} + +#[test] +fn legacy_yaml_boolean_words_are_not_enabled() { + for word in ["yes", "no", "on", "off"] { + assert!(load(&format!("exceptions: {{include_special: {word}}}")).is_err()); + } + for word in ["true", "True", "TRUE"] { + assert!( + load(&format!("exceptions: {{include_special: {word}}}")) + .unwrap() + .exceptions + .unwrap() + .include_special + ); + } +} diff --git a/src/lib/mlxcel-surgery/Cargo.toml b/src/lib/mlxcel-surgery/Cargo.toml index dcac967fb..988a267ef 100644 --- a/src/lib/mlxcel-surgery/Cargo.toml +++ b/src/lib/mlxcel-surgery/Cargo.toml @@ -20,14 +20,14 @@ rocm = ["mlxcel-core/rocm"] # `mlxcel::loading::vlm`. mlxcel-core = { path = "../mlxcel-core", default-features = false } -# Public API surfaces serde-derived types (builds on top by # adding `serde_yaml` for the YAML config schema). +# Public API surfaces serde-derived types; serde-saphyr parses the YAML config schema. serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" # YAML configuration schema parser. Pinned to the same version # the top-level `mlxcel` crate already uses to keep the dependency # graph small. -serde_yaml = "0.9" +serde-saphyr = { version = "1.3", default-features = false, features = ["deserialize"] } # Glob-style tensor key patterns. The same `globset` version is # transitively present via tracing-subscriber's fluent layer, so we diff --git a/src/lib/mlxcel-surgery/src/config.rs b/src/lib/mlxcel-surgery/src/config.rs index b43010b76..e565a1d0c 100644 --- a/src/lib/mlxcel-surgery/src/config.rs +++ b/src/lib/mlxcel-surgery/src/config.rs @@ -108,6 +108,7 @@ pub enum OpSpec { /// Glob pattern matched against tensor keys. pattern: String, /// Scalar multiplier (any finite f32). + #[serde(deserialize_with = "yaml_number::deserialize")] factor: f32, }, @@ -119,7 +120,10 @@ pub enum OpSpec { /// Path to a safetensors file containing the task vector. source: PathBuf, /// Coefficient on the source term. Defaults to `1.0`. - #[serde(default = "default_alpha_one")] + #[serde( + default = "default_alpha_one", + deserialize_with = "yaml_number::deserialize" + )] alpha: f32, }, @@ -162,6 +166,7 @@ pub enum OpSpec { /// Donor checkpoint B. source_b: PathBuf, /// Mixing ratio in `[0.0, 1.0]`. + #[serde(deserialize_with = "yaml_number::deserialize")] ratio: f32, /// Interpolation method. method: InterpolateMethod, @@ -212,7 +217,12 @@ pub fn parse_config_str( yaml: &str, base_dir: Option<&Path>, ) -> Result { - let config: SurgeryConfig = serde_yaml::from_str(yaml) + let options = serde_saphyr::options! { + merge_keys: serde_saphyr::MergeKeyPolicy::AsOrdinary, + strict_booleans: true, + reject_non_finite_typeless_float: false, + }; + let config: SurgeryConfig = serde_saphyr::from_str_with_options(yaml, options) .map_err(|e| SurgeryError::Other(anyhow::anyhow!("failed to parse surgery YAML: {e}")))?; if config.version != SUPPORTED_SCHEMA_VERSION { @@ -886,3 +896,10 @@ operations: parse_tempfile(yaml) } } + +#[cfg(test)] +#[path = "config_yaml_compat_tests.rs"] +mod yaml_compat_tests; + +#[path = "config_yaml_number.rs"] +mod yaml_number; diff --git a/src/lib/mlxcel-surgery/src/config_yaml_compat_tests.rs b/src/lib/mlxcel-surgery/src/config_yaml_compat_tests.rs new file mode 100644 index 000000000..c099f12ea --- /dev/null +++ b/src/lib/mlxcel-surgery/src/config_yaml_compat_tests.rs @@ -0,0 +1,100 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +#[test] +fn duplicate_schema_fields_are_rejected() { + for yaml in [ + "version: 1\nversion: 1\noperations: []\n", + "version: 1\noperations: [{op: scale, pattern: '*', factor: 1, factor: 2}]\n", + ] { + assert!(parse_config_str(yaml, None).is_err()); + } +} + +#[test] +fn shipped_examples_keep_their_enum_and_numeric_syntax() { + let options = || { + serde_saphyr::options! { + merge_keys: serde_saphyr::MergeKeyPolicy::AsOrdinary, + strict_booleans: true, + reject_non_finite_typeless_float: false, + } + }; + for yaml in [ + include_str!("../../../../examples/surgery/add_example.yaml"), + include_str!("../../../../examples/surgery/full_example.yaml"), + include_str!("../../../../examples/surgery/interpolate_slerp_example.yaml"), + include_str!("../../../../examples/surgery/prune_head_example.yaml"), + include_str!("../../../../examples/surgery/replace_example.yaml"), + include_str!("../../../../examples/surgery/scale_only.yaml"), + ] { + let config: SurgeryConfig = serde_saphyr::from_str_with_options(yaml, options()).unwrap(); + assert_eq!(config.version, SUPPORTED_SCHEMA_VERSION); + assert!(!config.operations.is_empty()); + } +} + +#[test] +fn non_finite_factors_reach_operation_validation() { + for value in [ + ".nan", ".NaN", ".inf", "-.inf", "+.Inf", "'.nan'", "'.inf'", "'-.inf'", + ] { + let yaml = + format!("version: 1\noperations: [{{op: scale, pattern: '*', factor: {value}}}]"); + let error = parse_config_str(&yaml, None) + .err() + .expect("non-finite rejected") + .to_string(); + assert!( + error.contains("scale") && error.contains("finite"), + "{value}: {error}" + ); + } +} + +#[test] +fn non_finite_coefficients_reach_add_validation() { + for value in [".nan", ".inf", "-.inf", "'.nan'", "'.inf'", "'-.inf'"] { + let yaml = format!( + "version: 1\noperations: [{{op: add, pattern: '*', source: donor.safetensors, alpha: {value}}}]" + ); + let error = parse_config_str(&yaml, None) + .err() + .expect("non-finite rejected") + .to_string(); + assert!( + error.contains("(add)") && error.contains("alpha must be finite"), + "{value}: {error}" + ); + } +} + +#[test] +fn non_finite_ratios_reach_interpolate_validation() { + for value in [".nan", ".inf", "-.inf", "'.nan'", "'.inf'", "'-.inf'"] { + let yaml = format!( + "version: 1\noperations: [{{op: interpolate, pattern: '*', source_a: a.safetensors, source_b: b.safetensors, method: lerp, ratio: {value}}}]" + ); + let error = parse_config_str(&yaml, None) + .err() + .expect("non-finite rejected") + .to_string(); + assert!( + error.contains("(interpolate)") && error.contains("ratio must be a finite number"), + "{value}: {error}" + ); + } +} diff --git a/src/lib/mlxcel-surgery/src/config_yaml_number.rs b/src/lib/mlxcel-surgery/src/config_yaml_number.rs new file mode 100644 index 000000000..358a24694 --- /dev/null +++ b/src/lib/mlxcel-surgery/src/config_yaml_number.rs @@ -0,0 +1,42 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Preserve non-finite values through Serde's internally tagged enum buffer. +//! +//! serde-saphyr represents typeless non-finite floats as canonical strings. +//! Recover only those spellings so the existing operation validators can emit +//! their specific finite-value errors; ordinary quoted numbers remain invalid. + +use serde::Deserialize; + +pub(super) fn deserialize<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum Number { + Float(f32), + Special(String), + } + match Number::deserialize(deserializer)? { + Number::Float(value) => Ok(value), + Number::Special(value) => match value.as_str() { + ".nan" => Ok(f32::NAN), + ".inf" => Ok(f32::INFINITY), + "-.inf" => Ok(f32::NEG_INFINITY), + _ => Err(serde::de::Error::custom("expected a number")), + }, + } +}