From 1e6c106e85dfd26569edcab935e194c39bb8bf0b Mon Sep 17 00:00:00 2001 From: benbenbang Date: Sat, 14 Nov 2026 21:25:01 -0500 Subject: [PATCH 1/5] fix(math): port Boost lanczos13m53 lanczos sum into beta U1 of #1468. Adds the private `lanczos_sum_exp_g_scaled` to `src/math/beta.rs`, adapted from Boost.Math 1.92 `boost/math/special_functions/lanczos.hpp` (`lanczos13m53`, Copyright John Maddock 2006). It evaluates the rational function as Boost's `evaluate_rational`: in `z` for `z <= 1` and in `1/z` above, so large `z` cannot overflow. The function is marked `expect(dead_code)` outside tests until U2 consumes it. A new mpmath oracle at 60 digits (`tests/fixtures/ibeta_power_terms/generate.py` and `cases.csv`) pins the result on both arms of the rational evaluation to a relative tolerance of `1e-15`. Extends the Boost.Math third-party notice in both copies (`crates/libitofin/THIRD_PARTY_NOTICES.md` and the bundled Python copy) to cover `lanczos.hpp`. ref #1468 --- .../licenses/libitofin/THIRD_PARTY_NOTICES.md | 6 +- crates/libitofin/THIRD_PARTY_NOTICES.md | 6 +- crates/libitofin/src/math/beta.rs | 92 +++++++++++++++++++ .../fixtures/ibeta_power_terms/cases.csv | 15 +++ .../fixtures/ibeta_power_terms/generate.py | 41 +++++++++ 5 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv create mode 100644 crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py diff --git a/crates/itofin-py/python/itofin/licenses/libitofin/THIRD_PARTY_NOTICES.md b/crates/itofin-py/python/itofin/licenses/libitofin/THIRD_PARTY_NOTICES.md index 658884aa..759cb4b7 100644 --- a/crates/itofin-py/python/itofin/licenses/libitofin/THIRD_PARTY_NOTICES.md +++ b/crates/itofin-py/python/itofin/licenses/libitofin/THIRD_PARTY_NOTICES.md @@ -156,7 +156,7 @@ immutable reference-date and whole-year-tenor contracts, plus mixed native-oracl provenance, are documented in `docs/docs/cpi-capfloor-bindings.md` at repository level and `sdk/go/testdata/cpi-capfloor-native.provenance.json`. -## Boost.Math Student-t quantile +## Boost.Math Student-t quantile and incomplete beta The Student-t quantile estimate in `src/math/distributions/studentt/inverse.rs` (closed forms for 1, 2 and 4 degrees of freedom, the 6 degrees of freedom @@ -164,6 +164,10 @@ iteration, Hill's method and Shaw's tail and body series) is adapted from Boost.Math 1.92, `boost/math/special_functions/detail/t_distribution_inv.hpp`, which QuantLib v1.44's `InverseCumulativeStudent` calls. Copyright John Maddock 2007; Copyright Paul A. Bristow 2007. + +The `lanczos13m53` Lanczos sum in `src/math/beta.rs` is adapted from +Boost.Math 1.92, `boost/math/special_functions/lanczos.hpp`. +Copyright John Maddock 2006. Distributed under the Boost Software License, Version 1.0: ```text diff --git a/crates/libitofin/THIRD_PARTY_NOTICES.md b/crates/libitofin/THIRD_PARTY_NOTICES.md index 658884aa..759cb4b7 100644 --- a/crates/libitofin/THIRD_PARTY_NOTICES.md +++ b/crates/libitofin/THIRD_PARTY_NOTICES.md @@ -156,7 +156,7 @@ immutable reference-date and whole-year-tenor contracts, plus mixed native-oracl provenance, are documented in `docs/docs/cpi-capfloor-bindings.md` at repository level and `sdk/go/testdata/cpi-capfloor-native.provenance.json`. -## Boost.Math Student-t quantile +## Boost.Math Student-t quantile and incomplete beta The Student-t quantile estimate in `src/math/distributions/studentt/inverse.rs` (closed forms for 1, 2 and 4 degrees of freedom, the 6 degrees of freedom @@ -164,6 +164,10 @@ iteration, Hill's method and Shaw's tail and body series) is adapted from Boost.Math 1.92, `boost/math/special_functions/detail/t_distribution_inv.hpp`, which QuantLib v1.44's `InverseCumulativeStudent` calls. Copyright John Maddock 2007; Copyright Paul A. Bristow 2007. + +The `lanczos13m53` Lanczos sum in `src/math/beta.rs` is adapted from +Boost.Math 1.92, `boost/math/special_functions/lanczos.hpp`. +Copyright John Maddock 2006. Distributed under the Boost Software License, Version 1.0: ```text diff --git a/crates/libitofin/src/math/beta.rs b/crates/libitofin/src/math/beta.rs index 6575ee67..7a2f349b 100644 --- a/crates/libitofin/src/math/beta.rs +++ b/crates/libitofin/src/math/beta.rs @@ -9,6 +9,11 @@ //! accuracy (`1e-16`) but its iteration limit grows with the parameters (see //! [`incomplete_beta`]), where QuantLib's fixed limit of 100 throws for large //! `a` or `b` (#1463). +//! +//! The Lanczos sum is adapted from Boost.Math 1.92 +//! `boost/math/special_functions/lanczos.hpp` (`lanczos13m53`, Copyright John +//! Maddock 2006, distributed under the Boost Software License 1.0, see +//! `THIRD_PARTY_NOTICES.md`). use crate::errors::QlResult; use crate::fail; @@ -19,6 +24,42 @@ const ACCURACY: Real = 1e-16; const MIN_ITERATIONS: u32 = 100; const MAX_ITERATIONS: u32 = 100_000_000; +/// Numerator coefficients of `lanczos13m53::lanczos_sum_expG_scaled`. +#[allow(clippy::excessive_precision)] +const LANCZOS_NUM: [Real; 13] = [ + 56906521.91347156388090791033559122686859, + 103794043.1163445451906271053616070238554, + 86363131.28813859145546927288977868422342, + 43338889.32467613834773723740590533316085, + 14605578.08768506808414169982791359218571, + 3481712.15498064590882071018964774556468, + 601859.6171681098786670226533699352302507, + 75999.29304014542649875303443598909137092, + 6955.999602515376140356310115515198987526, + 449.9445569063168119446858607650988409623, + 19.51992788247617482847860966235652136208, + 0.5098416655656676188125178644804694509993, + 0.006061842346248906525783753964555936883222, +]; + +/// Denominator coefficients of `lanczos13m53` (the expansion of +/// `z (z + 1) ... (z + 11)`). +const LANCZOS_DENOM: [Real; 13] = [ + 0.0, + 39916800.0, + 120543840.0, + 150917976.0, + 105258076.0, + 45995730.0, + 13339535.0, + 2637558.0, + 357423.0, + 32670.0, + 1925.0, + 66.0, + 1.0, +]; + /// The complete beta function `B(z, w) = Γ(z)·Γ(w) / Γ(z+w)`. /// /// # Errors @@ -107,6 +148,33 @@ pub fn incomplete_beta(a: Real, b: Real, x: Real) -> QlResult { } } +/// Boost's `lanczos13m53::lanczos_sum_expG_scaled(z)`, the Lanczos sum +/// `Γ(z) e^(z - 1/2) / (z + g - 1/2)^(z - 1/2)` for `z > 0`. +/// +/// Evaluates the rational function as Boost's `tools::evaluate_rational`: in +/// `z` for `z <= 1`, and in `1/z` above, so large `z` cannot overflow. +#[cfg_attr(not(test), expect(dead_code))] +fn lanczos_sum_exp_g_scaled(z: Real) -> Real { + let (mut num, mut denom); + if z <= 1.0 { + num = LANCZOS_NUM[12]; + denom = LANCZOS_DENOM[12]; + for i in (0..12).rev() { + num = num * z + LANCZOS_NUM[i]; + denom = denom * z + LANCZOS_DENOM[i]; + } + } else { + let z = 1.0 / z; + num = LANCZOS_NUM[0]; + denom = LANCZOS_DENOM[0]; + for i in 1..13 { + num = num * z + LANCZOS_NUM[i]; + denom = denom * z + LANCZOS_DENOM[i]; + } + } + num / denom +} + /// Lentz's continued-fraction evaluation for [`incomplete_beta`] (NR `betacf`). fn beta_continued_fraction(a: Real, b: Real, x: Real) -> QlResult { let eps = Real::EPSILON; @@ -219,6 +287,30 @@ mod tests { } } + /// mpmath at 60 digits (`tests/fixtures/ibeta_power_terms/generate.py`), + /// on both arms of the rational evaluation (`z <= 1` and `z > 1`). + #[test] + fn lanczos_sum_exp_g_scaled_matches_mpmath() { + let rows: Vec<(Real, Real)> = + include_str!("../../tests/fixtures/ibeta_power_terms/cases.csv") + .lines() + .filter_map(|line| line.strip_prefix("lanczos,")) + .map(|line| { + let (z, value) = line.split_once(',').unwrap(); + (z.parse().unwrap(), value.parse().unwrap()) + }) + .collect(); + assert_eq!(rows.len(), 14); + for (z, expected) in rows { + let got = lanczos_sum_exp_g_scaled(z); + let rel = ((got - expected) / expected).abs(); + assert!( + rel <= 1e-15, + "z={z}: got {got}, expected {expected}, rel {rel:e}" + ); + } + } + #[test] fn invalid_args_rejected() { assert!(incomplete_beta(0.0, 1.0, 0.5).is_err()); // a <= 0 diff --git a/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv b/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv new file mode 100644 index 00000000..3336df6e --- /dev/null +++ b/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv @@ -0,0 +1,15 @@ +# mpmath 1.3.0, dps 60; kind,inputs...,value +lanczos,0.001,1.4239253119544977e+3 +lanczos,0.25,4.3771299076064442 +lanczos,0.5,1.772453850905516 +lanczos,1.0,6.4545746981950712e-1 +lanczos,1.5,3.4293583954935442e-1 +lanczos,2.0,2.1712491729093509e-1 +lanczos,3.5,9.0816385665194957e-2 +lanczos,10.0,2.3487963185312735e-2 +lanczos,47.25,8.6615609203608131e-3 +lanczos,1000.0,6.1722097041602029e-3 +lanczos,1000000.0,6.0619521069263376e-3 +lanczos,500000000000.0,6.0618423464684272e-3 +lanczos,5000000000000000.0,6.061842346248929e-3 +lanczos,1e+20,6.0618423462489071e-3 diff --git a/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py b/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py new file mode 100644 index 00000000..af1ac2bf --- /dev/null +++ b/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py @@ -0,0 +1,41 @@ +"""Oracle for libitofin #1468 (incomplete beta power terms at large a, b). + +Rows are computed with mpmath at 60 significant digits and rounded to 17; +inputs are binary64 values printed exactly. + +- `lanczos,z,value`: Boost's `lanczos13m53::lanczos_sum_expG_scaled(z)`, the + exact `Gamma(z) e^(z - 1/2) / (z + g - 1/2)^(z - 1/2)` at Boost's binary64 g. + +Regenerate from the repository root: + + uv run --with mpmath==1.3.0 python -I \ + crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py \ + > crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv +""" + +import mpmath + +mpmath.mp.dps = 60 + +LANCZOS_G = mpmath.mpf(6.024680040776729583740234375) +LANCZOS_Z = [1e-3, 0.25, 0.5, 1.0, 1.5, 2.0, 3.5, 10.0, 47.25, 1e3, 1e6, 5e11, 5e15, 1e20] + + +def lanczos_scaled(z): + z = mpmath.mpf(z) + gh = LANCZOS_G - mpmath.mpf(1) / 2 + return mpmath.exp(mpmath.loggamma(z) + z - mpmath.mpf(1) / 2 - (z - mpmath.mpf(1) / 2) * mpmath.log(z + gh)) + + +def fmt(x): + return mpmath.nstr(x, 17, min_fixed=-1, max_fixed=1) + + +def main(): + print(f"# mpmath {mpmath.__version__}, dps {mpmath.mp.dps}; kind,inputs...,value") + for z in LANCZOS_Z: + print(f"lanczos,{z!r},{fmt(lanczos_scaled(z))}") + + +if __name__ == "__main__": + main() From 58ba1764a1302d6ca510b3712ccff50598287c1f Mon Sep 17 00:00:00 2001 From: benbenbang Date: Sat, 14 Nov 2026 21:48:44 -0500 Subject: [PATCH 2/5] fix(math): replace cancelling incomplete_beta prefactor with Boost power terms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit U2 of #1468. QuantLib's `exp(lgamma(a+b) - lgamma(a) - lgamma(b) + a ln x + b ln(1-x))` prefactor cancels for large `a`, `b` (the Binomial cdf at `n = 1e16` returned only 0 or 1). Port Boost 1.92's normalised `ibeta_power_terms(a, b, x, y)` as the prefactor instead, carrying `Γ(a+b)/(Γ(a)Γ(b))` in `lanczos13m53` sums and forming near-1 powers via `log1p` rather than cancelling logarithms. All branches of the normalised arm are ported; where Boost raises an underflow or overflow it returns the product here, so a deep tail evaluates to `0` rather than `Err`. **Incomplete beta changes:** * Add `pub(crate) incomplete_beta_xy(a, b, x, y)`: the caller supplies the complement `y = 1 - x` (often exact where `1 - x` would round), used in the power terms and the swapped continued fraction; the branch test stays on `x`. `incomplete_beta` keeps its signature and delegates. * Guard `y` to `[0, 1]`; return 1 when `x == 1.0 || y == 0.0`. * Promote `lanczos_sum_exp_g_scaled` out of test-only `dead_code`. **Tests and oracle:** * Add `incomplete_beta_power_terms.rs` and `binomial` fixture rows for `n = 1e12, 1e14, 1e16` at `p = 1/2` and `n = 1e12` at `p = 3/4`, generated by mpmath log-space tanh-sinh quadrature (betainc does not return at these sizes), pinned at `5e-10`, `1e-8`, `5e-7`: the continued fraction still cancels near the saddle beyond about `1e11` (Boost 1.92 itself misses `1e-10`), deferred to a large-parameter asymptotic. * Tighten `incomplete_beta_large_a` binomial tolerance `1e-9` -> `1e-12`; the power terms bring those rows within `7e-14`. **Docs and notices:** * Document the prefactor swap and the saddle deferral in `beta.rs` and `wiki/compatibility.md`. * Extend the Boost notices to `beta.hpp` (Copyright Matt Borland 2024). ref #1468 --- .../licenses/libitofin/THIRD_PARTY_NOTICES.md | 7 +- crates/libitofin/THIRD_PARTY_NOTICES.md | 7 +- crates/libitofin/src/math/beta.rs | 164 ++++++++++++++++-- .../fixtures/ibeta_power_terms/cases.csv | 20 +++ .../fixtures/ibeta_power_terms/generate.py | 37 ++++ .../tests/incomplete_beta_large_a.rs | 8 +- .../tests/incomplete_beta_power_terms.rs | 44 +++++ wiki/compatibility.md | 10 +- 8 files changed, 269 insertions(+), 28 deletions(-) create mode 100644 crates/libitofin/tests/incomplete_beta_power_terms.rs diff --git a/crates/itofin-py/python/itofin/licenses/libitofin/THIRD_PARTY_NOTICES.md b/crates/itofin-py/python/itofin/licenses/libitofin/THIRD_PARTY_NOTICES.md index 759cb4b7..9596c8a2 100644 --- a/crates/itofin-py/python/itofin/licenses/libitofin/THIRD_PARTY_NOTICES.md +++ b/crates/itofin-py/python/itofin/licenses/libitofin/THIRD_PARTY_NOTICES.md @@ -165,9 +165,10 @@ Boost.Math 1.92, `boost/math/special_functions/detail/t_distribution_inv.hpp`, which QuantLib v1.44's `InverseCumulativeStudent` calls. Copyright John Maddock 2007; Copyright Paul A. Bristow 2007. -The `lanczos13m53` Lanczos sum in `src/math/beta.rs` is adapted from -Boost.Math 1.92, `boost/math/special_functions/lanczos.hpp`. -Copyright John Maddock 2006. +The incomplete beta power terms (`ibeta_power_terms`) and the `lanczos13m53` +Lanczos sum in `src/math/beta.rs` are adapted from Boost.Math 1.92, +`boost/math/special_functions/beta.hpp` and `lanczos.hpp`. +Copyright John Maddock 2006; Copyright Matt Borland 2024. Distributed under the Boost Software License, Version 1.0: ```text diff --git a/crates/libitofin/THIRD_PARTY_NOTICES.md b/crates/libitofin/THIRD_PARTY_NOTICES.md index 759cb4b7..9596c8a2 100644 --- a/crates/libitofin/THIRD_PARTY_NOTICES.md +++ b/crates/libitofin/THIRD_PARTY_NOTICES.md @@ -165,9 +165,10 @@ Boost.Math 1.92, `boost/math/special_functions/detail/t_distribution_inv.hpp`, which QuantLib v1.44's `InverseCumulativeStudent` calls. Copyright John Maddock 2007; Copyright Paul A. Bristow 2007. -The `lanczos13m53` Lanczos sum in `src/math/beta.rs` is adapted from -Boost.Math 1.92, `boost/math/special_functions/lanczos.hpp`. -Copyright John Maddock 2006. +The incomplete beta power terms (`ibeta_power_terms`) and the `lanczos13m53` +Lanczos sum in `src/math/beta.rs` are adapted from Boost.Math 1.92, +`boost/math/special_functions/beta.hpp` and `lanczos.hpp`. +Copyright John Maddock 2006; Copyright Matt Borland 2024. Distributed under the Boost Software License, Version 1.0: ```text diff --git a/crates/libitofin/src/math/beta.rs b/crates/libitofin/src/math/beta.rs index 7a2f349b..9300a069 100644 --- a/crates/libitofin/src/math/beta.rs +++ b/crates/libitofin/src/math/beta.rs @@ -2,18 +2,23 @@ //! //! Port of `ql/math/beta.{hpp,cpp}`: the complete [`beta_function`] and the //! regularized [`incomplete_beta`] (the CDF of the Beta distribution), the -//! latter via the Numerical Recipes continued-fraction expansion with a -//! [`log_gamma`] prefactor. Needed for the Student-t CDF. +//! latter via the Numerical Recipes continued-fraction expansion. Needed for +//! the Student-t CDF. //! -//! Divergence from QuantLib: the continued fraction keeps QuantLib's +//! Divergences from QuantLib: the continued fraction keeps QuantLib's //! accuracy (`1e-16`) but its iteration limit grows with the parameters (see //! [`incomplete_beta`]), where QuantLib's fixed limit of 100 throws for large -//! `a` or `b` (#1463). +//! `a` or `b` (#1463); and its prefactor `x^a y^b / B(a, b)` is Boost's +//! `ibeta_power_terms`, where QuantLib's `exp(lgamma)` form cancels for large +//! `a`, `b` (#1468). //! -//! The Lanczos sum is adapted from Boost.Math 1.92 -//! `boost/math/special_functions/lanczos.hpp` (`lanczos13m53`, Copyright John -//! Maddock 2006, distributed under the Boost Software License 1.0, see -//! `THIRD_PARTY_NOTICES.md`). +//! The power terms and the Lanczos sum are adapted from Boost.Math 1.92 +//! `boost/math/special_functions/beta.hpp` (Copyright John Maddock 2006, +//! Copyright Matt Borland 2024) and `lanczos.hpp` (`lanczos13m53`, Copyright +//! John Maddock 2006), distributed under the Boost Software License 1.0, see +//! `THIRD_PARTY_NOTICES.md`. + +use std::f64::consts::E; use crate::errors::QlResult; use crate::fail; @@ -24,6 +29,10 @@ const ACCURACY: Real = 1e-16; const MIN_ITERATIONS: u32 = 100; const MAX_ITERATIONS: u32 = 100_000_000; +/// The `g` of Boost's `lanczos13m53` approximation (`lanczos.hpp`). +#[allow(clippy::excessive_precision)] +const LANCZOS_G: Real = 6.024680040776729583740234375; + /// Numerator coefficients of `lanczos13m53::lanczos_sum_expG_scaled`. #[allow(clippy::excessive_precision)] const LANCZOS_NUM: [Real; 13] = [ @@ -98,8 +107,16 @@ pub fn beta_function(z: Real, w: Real) -> QlResult { /// converged within 100 iterations exits at the same iteration with the same /// result. Past iteration 100 a `NaN` running product, which no later step can /// clear, returns a distinct `NaN` error at once rather than spinning to the -/// limit. Convergence is not accuracy: for very large `a`, `b` the `lgamma` -/// prefactor cancels and the result loses all precision (#1468). +/// limit. +/// +/// The prefactor is Boost's `ibeta_power_terms` (#1468), where QuantLib's +/// `exp(lgamma(a+b) - lgamma(a) - lgamma(b) + a ln x + b ln(1-x))` cancels +/// for large `a`, `b` (the Binomial cdf at `n = 1e16` returned only 0 or 1). +/// Deferred: near the saddle `x = a/(a+b)` the continued fraction itself +/// cancels for `a`, `b` beyond about `1e11`, so the Binomial cdf at +/// `n = 1e12, 1e14, 1e16` is within `5e-10`, `1e-8` and `5e-7` of mpmath +/// there, not `1e-10`; a large-parameter asymptotic expansion is tracked +/// separately. /// /// # Errors /// @@ -117,6 +134,19 @@ pub fn beta_function(z: Real, w: Real) -> QlResult { /// # Ok::<(), libitofin::errors::QlError>(()) /// ``` pub fn incomplete_beta(a: Real, b: Real, x: Real) -> QlResult { + incomplete_beta_xy(a, b, x, 1.0 - x) +} + +/// [`incomplete_beta`] with the complement `y = 1 - x` supplied by the +/// caller, who can often form it exactly where `1 - x` would round (the +/// Student-t cdf near `x = 1`). `y` enters the power terms and the swapped +/// continued fraction; the branch test stays on `x`, as in +/// [`incomplete_beta`]. +/// +/// # Errors +/// +/// As [`incomplete_beta`], and also if `y` is outside `[0, 1]`. +pub(crate) fn incomplete_beta_xy(a: Real, b: Real, x: Real, y: Real) -> QlResult { // `is_finite()` is required: a bare `<= 0.0` lets NaN and +infinity through // (all NaN comparisons are false, and +infinity is not <= 0), but // QuantLib's QL_REQUIRE(a > 0) rejects them @@ -130,21 +160,23 @@ pub fn incomplete_beta(a: Real, b: Real, x: Real) -> QlResult { if !(0.0..=1.0).contains(&x) { fail!("incomplete_beta requires x in [0, 1], got x={x}"); } + if !(0.0..=1.0).contains(&y) { + fail!("incomplete_beta requires y in [0, 1], got y={y}"); + } if x == 0.0 { return Ok(0.0); } - if x == 1.0 { + if x == 1.0 || y == 0.0 { return Ok(1.0); } - let prefactor = - (log_gamma(a + b)? - log_gamma(a)? - log_gamma(b)? + a * x.ln() + b * (1.0 - x).ln()).exp(); + let prefactor = ibeta_power_terms(a, b, x, y); // use the expansion that converges fastest for this x if x < (a + 1.0) / (a + b + 2.0) { Ok(prefactor * beta_continued_fraction(a, b, x)? / a) } else { - Ok(1.0 - prefactor * beta_continued_fraction(b, a, 1.0 - x)? / b) + Ok(1.0 - prefactor * beta_continued_fraction(b, a, y)? / b) } } @@ -153,7 +185,6 @@ pub fn incomplete_beta(a: Real, b: Real, x: Real) -> QlResult { /// /// Evaluates the rational function as Boost's `tools::evaluate_rational`: in /// `z` for `z <= 1`, and in `1/z` above, so large `z` cannot overflow. -#[cfg_attr(not(test), expect(dead_code))] fn lanczos_sum_exp_g_scaled(z: Real) -> Real { let (mut num, mut denom); if z <= 1.0 { @@ -175,6 +206,109 @@ fn lanczos_sum_exp_g_scaled(z: Real) -> Real { num / denom } +/// Boost's normalised `ibeta_power_terms(a, b, x, y)`: the leading factor +/// `x^a y^b / B(a, b)` of the incomplete beta, from `y` as given (not +/// `1 - x`), for `a, b > 0` and `x, y` in `(0, 1)`. +/// +/// The Lanczos sums carry `Γ(a + b) / (Γ(a) Γ(b))` and the bases of the two +/// powers are rescaled by `(a + b + g - 1/2) / (a + g - 1/2)` and its `b` +/// analogue, so where a base is near 1 the power is formed as +/// `exp(a log1p(l))` rather than from cancelling logarithms. Boost raises an +/// underflow or overflow error where the product is `0` or `∞`; the product +/// is returned instead, so a deep tail evaluates to `0`. +fn ibeta_power_terms(a: Real, b: Real, x: Real, y: Real) -> Real { + let c = a + b; + let gh = LANCZOS_G - 0.5; + let agh = a + gh; + let bgh = b + gh; + let cgh = c + gh; + let log_max = Real::MAX.ln(); + let log_min = Real::MIN_POSITIVE.ln(); + let mut result = if a < Real::MIN_POSITIVE || b < Real::MIN_POSITIVE { + 0.0 + } else { + lanczos_sum_exp_g_scaled(c) / (lanczos_sum_exp_g_scaled(a) * lanczos_sum_exp_g_scaled(b)) + }; + result *= (bgh / E).sqrt(); + result *= (agh / cgh).sqrt(); + + let l1 = ((x * b - y * a) - y * gh) / agh; + let l2 = ((y * a - x * b) - x * gh) / bgh; + if l1.abs().min(l2.abs()) < 0.2 { + if l1 * l2 > 0.0 || a.min(b) < 1.0 { + result *= if l1.abs() < 0.1 { + (a * l1.ln_1p()).exp() + } else { + (x * cgh / agh).powf(a) + }; + result *= if l2.abs() < 0.1 { + (b * l2.ln_1p()).exp() + } else { + (y * cgh / bgh).powf(b) + }; + } else if l1.abs().max(l2.abs()) < 0.5 { + let ratio = b / a; + if (a < b && ratio * l2 < 0.1) || (a >= b && l1 / ratio > 0.1) { + let l3 = (ratio * l2.ln_1p()).exp_m1(); + result *= (a * (l1 + l3 + l3 * l1).ln_1p()).exp(); + } else { + let l3 = (l1.ln_1p() / ratio).exp_m1(); + result *= (b * (l2 + l3 + l3 * l2).ln_1p()).exp(); + } + } else { + let l = if l1.abs() < l2.abs() { + a * l1.ln_1p() + b * (y * cgh / bgh).ln() + } else { + b * l2.ln_1p() + a * (x * cgh / agh).ln() + }; + result = if l <= log_min || l >= log_max { + (l + result.ln()).exp() + } else { + result * l.exp() + }; + } + } else { + let b1 = x * cgh / agh; + let b2 = y * cgh / bgh; + let l1 = a * b1.ln(); + let l2 = b * b2.ln(); + if l1 >= log_max || l1 <= log_min || l2 >= log_max || l2 <= log_min { + if a < b { + let p1 = b2.powf(b / a); + let l3 = if b1 != 0.0 && p1 != 0.0 { + a * (b1.ln() + p1.ln()) + } else { + Real::MAX + }; + result = if l3 < log_max && l3 > log_min { + result * (p1 * b1).powf(a) + } else { + (l2 + l1 + result.ln()).exp() + }; + } else { + let p1 = if b1 < 1.0 && b < 1.0 && Real::MAX * b < a { + 0.0 + } else { + b1.powf(a / b) + }; + let l3 = if p1 != 0.0 && b2 != 0.0 { + (p1.ln() + b2.ln()) * b + } else { + Real::MAX + }; + if l3 < log_max && l3 > log_min { + result *= (p1 * b2).powf(b); + } else if result != 0.0 { + result = (l2 + l1 + result.ln()).exp(); + } + } + } else { + result *= b1.powf(a) * b2.powf(b); + } + } + result +} + /// Lentz's continued-fraction evaluation for [`incomplete_beta`] (NR `betacf`). fn beta_continued_fraction(a: Real, b: Real, x: Real) -> QlResult { let eps = Real::EPSILON; diff --git a/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv b/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv index 3336df6e..e468ac77 100644 --- a/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv +++ b/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv @@ -13,3 +13,23 @@ lanczos,1000000.0,6.0619521069263376e-3 lanczos,500000000000.0,6.0618423464684272e-3 lanczos,5000000000000000.0,6.061842346248929e-3 lanczos,1e+20,6.0618423462489071e-3 +binomial,1000000000000,0.5,499999500000,1.5865549590230256e-1 +binomial,1000000000000,0.5,499999750000,3.0853789079141268e-1 +binomial,1000000000000,0.5,500000000000,5.000003989422804e-1 +binomial,1000000000000,0.5,500000250000,6.9146281333924085e-1 +binomial,1000000000000,0.5,500000500000,8.4134498803914648e-1 +binomial,100000000000000,0.5,49999995000000,1.5865527812853071e-1 +binomial,100000000000000,0.5,49999997500000,3.0853757393252056e-1 +binomial,100000000000000,0.5,50000000000000,5.0000003989422804e-1 +binomial,100000000000000,0.5,50000002500000,6.9146249648054479e-1 +binomial,100000000000000,0.5,50000005000000,8.4134477026561419e-1 +binomial,10000000000000000,0.5,4999999950000000,1.5865525635116431e-1 +binomial,10000000000000000,0.5,4999999975000000,3.0853754224664017e-1 +binomial,10000000000000000,0.5,5000000000000000,5.000000039894228e-1 +binomial,10000000000000000,0.5,5000000025000000,6.9146246479466636e-1 +binomial,10000000000000000,0.5,5000000050000000,8.4134474848825018e-1 +binomial,1000000000000,0.75,749999566987,1.5865536675030309e-1 +binomial,1000000000000,0.75,749999783494,3.0853817978018819e-1 +binomial,1000000000000,0.75,750000000000,5.000003838823883e-1 +binomial,1000000000000,0.75,750000216506,6.9146253164752941e-1 +binomial,1000000000000,0.75,750000433013,8.4134519205689218e-1 diff --git a/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py b/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py index af1ac2bf..ac446fcc 100644 --- a/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py +++ b/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py @@ -5,6 +5,17 @@ - `lanczos,z,value`: Boost's `lanczos13m53::lanczos_sum_expG_scaled(z)`, the exact `Gamma(z) e^(z - 1/2) / (z + g - 1/2)^(z - 1/2)` at Boost's binary64 g. +- `binomial,n,p,k,P`: `P(X <= k)` for `X ~ Binomial(n, p)`, that is + `I_{1-p}(n - k, k + 1)` at the binary64 roundings of `n - k` and `k + 1` + (the arguments the Rust cdf passes), for `k` at `-2..=2` half standard + deviations around the mean. `p` is 1/2 or 3/4, so `1 - p` is exact in + binary64. mpmath's `betainc` does not return at these sizes, so the + integral of the Beta density is taken directly: tanh-sinh quadrature of + `exp((a-1) ln t + (b-1) ln(1-t) - ln B(a, b))` over a window of 60 + standard deviations below the mode, split at every standard deviation. + The route matches exact integer binomial sums at `n = 100000` to `1e-35` + and the complement integral `1 - I_p(k + 1, n - k)` at `n = 1e16` to + `1e-32`. Regenerate from the repository root: @@ -27,6 +38,28 @@ def lanczos_scaled(z): return mpmath.exp(mpmath.loggamma(z) + z - mpmath.mpf(1) / 2 - (z - mpmath.mpf(1) / 2) * mpmath.log(z + gh)) +BINOMIAL = [(10**12, 0.5), (10**14, 0.5), (10**16, 0.5), (10**12, 0.75)] + + +def ibeta_quad(a, b, x): + a, b, x = mpmath.mpf(a), mpmath.mpf(b), mpmath.mpf(x) + lnb = mpmath.loggamma(a) + mpmath.loggamma(b) - mpmath.loggamma(a + b) + mode = (a - 1) / (a + b - 2) + sigma = mpmath.sqrt(a * b / ((a + b) ** 2 * (a + b + 1))) + + def density(t): + return mpmath.exp((a - 1) * mpmath.log(t) + (b - 1) * mpmath.log1p(-t) - lnb) + + lo = max(mpmath.mpf(0), mode - 60 * sigma) + pieces = int(mpmath.ceil((x - lo) / sigma)) + 1 + return mpmath.quad(density, [lo + (x - lo) * i / pieces for i in range(pieces + 1)]) + + +def binomial_ks(n, p): + mean, sd = n * mpmath.mpf(p), mpmath.sqrt(n * mpmath.mpf(p) * (1 - mpmath.mpf(p))) + return [int(mpmath.nint(mean + j * sd / 2)) for j in range(-2, 3)] + + def fmt(x): return mpmath.nstr(x, 17, min_fixed=-1, max_fixed=1) @@ -35,6 +68,10 @@ def main(): print(f"# mpmath {mpmath.__version__}, dps {mpmath.mp.dps}; kind,inputs...,value") for z in LANCZOS_Z: print(f"lanczos,{z!r},{fmt(lanczos_scaled(z))}") + for n, p in BINOMIAL: + for k in binomial_ks(n, p): + value = ibeta_quad(float(n - k), float(k + 1), 1 - p) + print(f"binomial,{n},{p!r},{k},{fmt(value)}") if __name__ == "__main__": diff --git a/crates/libitofin/tests/incomplete_beta_large_a.rs b/crates/libitofin/tests/incomplete_beta_large_a.rs index 35a0bb4c..3670bcf7 100644 --- a/crates/libitofin/tests/incomplete_beta_large_a.rs +++ b/crates/libitofin/tests/incomplete_beta_large_a.rs @@ -11,11 +11,9 @@ use libitofin::types::Real; const REL_TOL: Real = 1e-10; -/// The Binomial prefactor sums `log_gamma` terms near `1e6` that cancel to -/// order one, so each ulp of them is about `1e-10` relative: over every `k` in -/// `[45000, 55000)` at `n = 100000` (converging ones included) the error peaks -/// at `2.1e-10`, independently of the iteration limit. -const BINOMIAL_REL_TOL: Real = 1e-9; +/// The `log_gamma` prefactor cancelled to `2.1e-10` here; Boost's power +/// terms (#1468) bring the rows within `7e-14`. +const BINOMIAL_REL_TOL: Real = 1e-12; fn cases(kind: &str) -> Vec<(Real, Real, Real)> { include_str!("fixtures/incomplete_beta_large_a/cases.csv") diff --git a/crates/libitofin/tests/incomplete_beta_power_terms.rs b/crates/libitofin/tests/incomplete_beta_power_terms.rs new file mode 100644 index 00000000..c17e0802 --- /dev/null +++ b/crates/libitofin/tests/incomplete_beta_power_terms.rs @@ -0,0 +1,44 @@ +//! #1468: the incomplete beta prefactor keeps its precision for very large +//! `a`, `b`, where the `log_gamma` form cancelled to garbage. Oracle: mpmath +//! at 60 digits (`fixtures/ibeta_power_terms/generate.py`). + +use libitofin::math::distributions::binomial::CumulativeBinomialDistribution; +use libitofin::types::Real; + +/// Per-`n` tolerance of the Binomial rows: the continued fraction cancels near +/// the saddle at these sizes (a documented deferral in `math/beta.rs`). +fn binomial_tol(n: u64) -> Real { + match n { + 1_000_000_000_000 => 5e-10, + 100_000_000_000_000 => 1e-8, + _ => 5e-7, + } +} + +fn rows(kind: &str) -> Vec> { + include_str!("fixtures/ibeta_power_terms/cases.csv") + .lines() + .filter_map(|line| line.strip_prefix(kind)) + .map(|line| line.split(',').map(|f| f.parse().unwrap()).collect()) + .collect() +} + +fn assert_rel(got: Real, expected: Real, tol: Real, context: &str) { + let err = ((got - expected) / expected).abs(); + assert!( + got.is_finite() && err <= tol, + "{context}: got {got}, expected {expected}, rel err {err:e}" + ); +} + +#[test] +fn binomial_cdf_at_huge_n_matches_mpmath() { + let rows = rows("binomial,"); + assert_eq!(rows.len(), 20); + for row in rows { + let (n, p, k, expected) = (row[0] as u64, row[1], row[2] as u64, row[3]); + let got = CumulativeBinomialDistribution::new(p, n).unwrap().cdf(k); + let context = format!("binomial n={n} p={p} k={k}"); + assert_rel(got, expected, binomial_tol(n), &context); + } +} diff --git a/wiki/compatibility.md b/wiki/compatibility.md index 90eebd93..8fe58a9f 100644 --- a/wiki/compatibility.md +++ b/wiki/compatibility.md @@ -115,8 +115,14 @@ oversight) and is documented at the point of divergence in the source. iterations-to-limit ratio seen over the swept Student-t and Binomial domains, bounds the work for absurd parameters, which return `Err`; past iteration 100 a `NaN` running product returns a distinct `Err` at once. - Convergence is not accuracy: for very large parameters the `lgamma` - prefactor cancels (#1468). + QuantLib's prefactor `exp(lgamma(a+b) - lgamma(a) - lgamma(b) + a ln x + + b ln(1-x))` cancels for very large parameters (the Binomial cdf at + `n = 1e16` returned only 0 or 1); here it is Boost's `ibeta_power_terms` + with the `lanczos13m53` sums (#1468). Near the saddle the continued fraction + still cancels beyond about `1e11`: the Binomial cdf at `p = 1/2` and + `n = 1e12, 1e14, 1e16` is within `5e-10`, `1e-8`, `5e-7` of mpmath (Boost + 1.92 itself is within `7e-11`, `3e-10`, `8e-9`), deferred to a + large-parameter asymptotic. - **The Student-t cdf evaluates the centre directly.** QuantLib's `CumulativeStudentDistribution` (`studenttdistribution.cpp`) always forms `I_x(ν/2, 1/2) / 2` with `x = ν/(ν + t²)`; near `t = 0` that `x` rounds From 72080cbf056e7d8a366e730926fe89553c69c3b3 Mon Sep 17 00:00:00 2001 From: benbenbang Date: Sat, 14 Nov 2026 22:10:08 -0500 Subject: [PATCH 3/5] fix(math): pass the exact incomplete-beta complement in Student-t cdf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Student-t cdf and `halley_step` now hand each incomplete-beta form its exact complement (`z` for `y`, `y` for `z`) through `incomplete_beta_xy` instead of `1 - z`, which rounds away the tail for large `ν`; `y` is `1` when `ν + t²` overflows. Beyond `ν = 1/eps` (`2^52`) the cdf returns the normal cdf, as Boost's `students_t` cdf does and unlike QuantLib's always taking the incomplete beta; the ticket's `1e20` limit lives only in the quantile. `incomplete_beta_xy`'s upper endpoint is `y == 0`, not `x == 1`: a rounded `z = 1.0` at `ν = 2^52`, `t = -0.5` reached the `x == 1` branch and returned `0.5`. The branch test stays on `x`. New oracle rows (mpmath tanh-sinh quadrature of the density) cover the cdf at `ν = 1e6, 1e12, 2^52, 2^52 + 2, 1e16` for centre `t` and the forward `t` where it is accurate (to `1e-10`), and the quantile at `ν = 1e9, 1e12` (to `1e-12`), with endpoint checks on the normal limit. Deferred and documented in `studentt.rs` and the wiki: for `|t| > √3` the forward continued fraction cancels for `ν ≥ 1e7` (about `1e-10` relative at `1e7`, `0.1` at `2^52`), where Boost uses `beta_small_b_large_a_series`. close #1468 --- crates/libitofin/src/math/beta.rs | 5 +- .../src/math/distributions/studentt.rs | 49 ++++++++++++------ .../fixtures/ibeta_power_terms/cases.csv | 34 +++++++++++++ .../fixtures/ibeta_power_terms/generate.py | 44 ++++++++++++++++ .../tests/incomplete_beta_power_terms.rs | 50 +++++++++++++++++++ wiki/compatibility.md | 7 +++ 6 files changed, 172 insertions(+), 17 deletions(-) diff --git a/crates/libitofin/src/math/beta.rs b/crates/libitofin/src/math/beta.rs index 9300a069..8a81d6ea 100644 --- a/crates/libitofin/src/math/beta.rs +++ b/crates/libitofin/src/math/beta.rs @@ -140,7 +140,8 @@ pub fn incomplete_beta(a: Real, b: Real, x: Real) -> QlResult { /// [`incomplete_beta`] with the complement `y = 1 - x` supplied by the /// caller, who can often form it exactly where `1 - x` would round (the /// Student-t cdf near `x = 1`). `y` enters the power terms and the swapped -/// continued fraction; the branch test stays on `x`, as in +/// continued fraction, and `y = 0` (not `x = 1`, which a rounded `x` can +/// reach first) is the upper endpoint; the branch test stays on `x`, as in /// [`incomplete_beta`]. /// /// # Errors @@ -166,7 +167,7 @@ pub(crate) fn incomplete_beta_xy(a: Real, b: Real, x: Real, y: Real) -> QlResult if x == 0.0 { return Ok(0.0); } - if x == 1.0 || y == 0.0 { + if y == 0.0 { return Ok(1.0); } diff --git a/crates/libitofin/src/math/distributions/studentt.rs b/crates/libitofin/src/math/distributions/studentt.rs index 9bdf09f8..f6177b35 100644 --- a/crates/libitofin/src/math/distributions/studentt.rs +++ b/crates/libitofin/src/math/distributions/studentt.rs @@ -3,7 +3,7 @@ //! Port of `ql/math/distributions/studenttdistribution.{hpp,cpp}`: the standard //! (location 0, scale 1) Student-t [`StudentT`] with `ν` degrees of freedom, //! providing the density, the cumulative distribution via the regularized -//! [`incomplete_beta`] function, and the inverse via Boost's Student-t quantile +//! incomplete beta function, and the inverse via Boost's Student-t quantile //! that QuantLib v1.44's `InverseCumulativeStudent` calls. use std::f64::consts::PI; @@ -11,7 +11,8 @@ use std::f64::consts::PI; use super::{Cdf, Density, Probability, Quantile, Support}; use crate::errors::QlResult; use crate::fail; -use crate::math::beta::incomplete_beta; +use crate::math::beta::incomplete_beta_xy; +use crate::math::distributions::normal::CumulativeNormalDistribution; use crate::math::gammafunction::log_gamma; use crate::require; use crate::types::Real; @@ -75,28 +76,42 @@ impl Cdf for StudentT { return Real::NAN; } let nu = self.degrees_of_freedom; + // Beyond 1/eps degrees of freedom the Student-t is the normal to + // within an ulp, and Boost's cdf returns the normal cdf there (#1468, a + // divergence: QuantLib always takes the incomplete beta). The 1e20 of + // the quantile is only where its normal estimate is taken as exact. + if nu > 1.0 / Real::EPSILON { + return CumulativeNormalDistribution::standard().value(x); + } let x2 = x * x; - let y = x2 / (nu + x2); + let total = nu + x2; + let y = if total.is_finite() { x2 / total } else { 1.0 }; + let z = nu / total; // Near the centre, where y = x²/(ν + x²) satisfies y·(ν+5) < 3, the // forward I_z(ν/2, 1/2) with z = ν/(ν + x²) rounds towards 1 and // cancels, so the half-mass I_y(1/2, ν/2) is evaluated directly (#1461, // a divergence from QuantLib's single forward form). The switch is - // incomplete_beta's own crossover for I_y(1/2, ν/2); y is NaN when x² + // incomplete_beta's own crossover for I_y(1/2, ν/2); y is 1 when ν + x² // overflows (|x| = ∞ included) and falls to the forward form. Either // way the arguments are a, b in {1/2, ν/2} with ν/2 > 0 and an argument // in [0, 1], and the 0 and 1 endpoints short-circuit in incomplete_beta - // before its continued fraction runs. Its iteration limit grows like + // before its continued fraction runs. Each form passes its exact + // complement (z for y, y for z), since 1 - z rounds away the tail for + // large ν (#1468). The incomplete beta's iteration limit grows like // sqrt(ν), so it converges on every swept input of this domain (#1463, // tests/incomplete_beta_large_a.rs; empirical, not a proof); a failure // is a defect in the special function, not a caller-recoverable - // condition, so we assert rather than propagate. + // condition, so we assert rather than propagate. Deferred: the forward + // form's continued fraction cancels at z near 1 for large ν (|x| > √3: + // about 1e-10 relative at ν = 1e7, 3e-5 at 1e12, 0.1 at 2^52), where + // Boost switches to beta_small_b_large_a_series; that port is tracked + // separately. const CONVERGES: &str = "incomplete beta must converge for valid Student-t parameters"; if y * (nu + 5.0) < 3.0 { - let half = 0.5 * incomplete_beta(0.5, 0.5 * nu, y).expect(CONVERGES); + let half = 0.5 * incomplete_beta_xy(0.5, 0.5 * nu, y, z).expect(CONVERGES); return if x <= 0.0 { 0.5 - half } else { 0.5 + half }; } - let z = nu / (nu + x2); - let tail = 0.5 * incomplete_beta(0.5 * nu, 0.5, z).expect(CONVERGES); + let tail = 0.5 * incomplete_beta_xy(0.5 * nu, 0.5, z, y).expect(CONVERGES); if x <= 0.0 { tail } else { 1.0 - tail } } @@ -128,11 +143,11 @@ impl StudentT { /// `(0.5 - u) - I_y(1/2, ν/2) / 2`. /// Boost switches at `x < y` and evaluates the complement /// `f0 = 1 - I_y(1/2, ν/2)` to full precision inside its ibeta; this - /// [`incomplete_beta`] has no complemented entry point, so `p0 = f0/2 - u` + /// incomplete beta has no complemented entry point, so `p0 = f0/2 - u` /// is rearranged to avoid the subtraction and used only where `I_y` is /// small (Boost's switch would cancel in the large-`ν` tails, where - /// `t² < ν` but `I_y` is near `1`). The pdf stands in for Boost's ibeta - /// derivative. + /// `t² < ν` but `I_y` is near `1`). Each form passes its exact complement, + /// as the cdf does. The pdf stands in for Boost's ibeta derivative. /// /// The step ratio `p0 / pdf(t)` is formed in log space, so it stays /// defined where the density is subnormal (the deep tail); only Boost's @@ -144,11 +159,15 @@ impl StudentT { if xb == 0.0 { return Ok(t); } - let y = t2 / (nu + t2); + let y = if (nu + t2).is_finite() { + t2 / (nu + t2) + } else { + 1.0 + }; let p0 = if y * (nu + 5.0) >= 3.0 { - 0.5 * incomplete_beta(0.5 * nu, 0.5, xb)? - u + 0.5 * incomplete_beta_xy(0.5 * nu, 0.5, xb, y)? - u } else { - (0.5 - u) - 0.5 * incomplete_beta(0.5, 0.5 * nu, y)? + (0.5 - u) - 0.5 * incomplete_beta_xy(0.5, 0.5 * nu, y, xb)? }; let r = p0.signum() * (p0.abs().ln() - self.ln_pdf(t)).exp(); let p2 = t * (nu + 1.0) / (t2 + nu); diff --git a/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv b/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv index e468ac77..f151c851 100644 --- a/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv +++ b/crates/libitofin/tests/fixtures/ibeta_power_terms/cases.csv @@ -33,3 +33,37 @@ binomial,1000000000000,0.75,749999783494,3.0853817978018819e-1 binomial,1000000000000,0.75,750000000000,5.000003838823883e-1 binomial,1000000000000,0.75,750000216506,6.9146253164752941e-1 binomial,1000000000000,0.75,750000433013,8.4134519205689218e-1 +student,1000000.0,-1.73,4.1815291884715043e-2 +student,1000000.0,-0.5,3.0853759373618569e-1 +student,1000000.0,0.5,6.9146240626381431e-1 +student,1000000.0,1.73,9.5818470811528496e-1 +student,1000000.0,-5.0,2.8669989354453708e-7 +student,1000000.0,-2.5,6.2097447510816231e-3 +student,1000000.0,-1.75,4.005931020827394e-2 +student,1000000.0,2.5,9.9379025524891838e-1 +student,1000000000000.0,-1.73,4.1815137613749221e-2 +student,1000000000000.0,-0.5,3.0853753872604191e-1 +student,1000000000000.0,0.5,6.9146246127395809e-1 +student,1000000000000.0,1.73,9.5818486238625078e-1 +student,4503599627370496.0,-1.73,4.1815137613594984e-2 +student,4503599627370496.0,-0.5,3.0853753872598691e-1 +student,4503599627370496.0,0.5,6.9146246127401309e-1 +student,4503599627370496.0,1.73,9.5818486238640502e-1 +student,4503599627370498.0,-1.73,4.1815137613594984e-2 +student,4503599627370498.0,-0.5,3.0853753872598691e-1 +student,4503599627370498.0,0.5,6.9146246127401309e-1 +student,4503599627370498.0,1.73,9.5818486238640502e-1 +student,4503599627370498.0,-5.0,2.8665157187920464e-7 +student,4503599627370498.0,-2.5,6.2096653257761528e-3 +student,4503599627370498.0,-1.75,4.0059156863817124e-2 +student,4503599627370498.0,2.5,9.9379033467422385e-1 +student,1e+16,-1.73,4.1815137613594965e-2 +student,1e+16,-0.5,3.085375387259869e-1 +student,1e+16,0.5,6.914624612740131e-1 +student,1e+16,1.73,9.5818486238640504e-1 +student,1e+16,-5.0,2.8665157187919874e-7 +student,1e+16,-2.5,6.2096653257761431e-3 +student,1e+16,-1.75,4.0059156863817106e-2 +student,1e+16,2.5,9.9379033467422386e-1 +student_quantile,1000000000.0,0.05,-1.6448536284752418 +student_quantile,1000000000000.0,0.05,-1.6448536269529965 diff --git a/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py b/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py index ac446fcc..60dd0844 100644 --- a/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py +++ b/crates/libitofin/tests/fixtures/ibeta_power_terms/generate.py @@ -16,6 +16,14 @@ The route matches exact integer binomial sums at `n = 100000` to `1e-35` and the complement integral `1 - I_p(k + 1, n - k)` at `n = 1e16` to `1e-32`. +- `student,nu,t,P`: the Student-t cdf, by tanh-sinh quadrature of the density + (with `loggamma`) from `-inf` to `t`; it matches mpmath's `betainc` at + `nu = 1e6, 1e12, 1e16` to `1e-33`. `nu` runs to either side of `2^52` + (`1/eps`), where the cdf switches to the normal. `|t| < sqrt(3)` takes + the centre form and `|t| > sqrt(3)` the forward one; the forward rows stop + at `nu = 1e6` below `2^52`, where the continued fraction still cancels + (a documented deferral in `studentt.rs`). +- `student_quantile,nu,p,x`: the Student-t quantile, by bisection on that cdf. Regenerate from the repository root: @@ -55,6 +63,36 @@ def density(t): return mpmath.quad(density, [lo + (x - lo) * i / pieces for i in range(pieces + 1)]) +STUDENT_NU = [1e6, 1e12, 4503599627370496.0, 4503599627370498.0, 1e16] +STUDENT_CENTRE_T = [-1.73, -0.5, 0.5, 1.73] +STUDENT_FORWARD_NU = [1e6, 4503599627370498.0, 1e16] +STUDENT_FORWARD_T = [-5.0, -2.5, -1.75, 2.5] +STUDENT_QUANTILE = [(1e9, 0.05), (1e12, 0.05)] + + +def student_cdf(nu, t): + nu, t = mpmath.mpf(nu), mpmath.mpf(t) + ln_c = mpmath.loggamma((nu + 1) / 2) - mpmath.loggamma(nu / 2) - mpmath.log(nu * mpmath.pi) / 2 + + def density(s): + return mpmath.exp(ln_c - (nu + 1) / 2 * mpmath.log1p(s * s / nu)) + + lower = min(t, -t) + tail = mpmath.quad(density, [-mpmath.inf, lower - 40, lower - 10, lower - 3, lower]) + return tail if t <= 0 else 1 - tail + + +def student_quantile(nu, p): + lo, hi = mpmath.mpf(-3), mpmath.mpf(0) + for _ in range(200): + mid = (lo + hi) / 2 + if student_cdf(nu, mid) < p: + lo = mid + else: + hi = mid + return (lo + hi) / 2 + + def binomial_ks(n, p): mean, sd = n * mpmath.mpf(p), mpmath.sqrt(n * mpmath.mpf(p) * (1 - mpmath.mpf(p))) return [int(mpmath.nint(mean + j * sd / 2)) for j in range(-2, 3)] @@ -72,6 +110,12 @@ def main(): for k in binomial_ks(n, p): value = ibeta_quad(float(n - k), float(k + 1), 1 - p) print(f"binomial,{n},{p!r},{k},{fmt(value)}") + for nu in STUDENT_NU: + ts = STUDENT_CENTRE_T + (STUDENT_FORWARD_T if nu in STUDENT_FORWARD_NU else []) + for t in ts: + print(f"student,{nu!r},{t!r},{fmt(student_cdf(nu, t))}") + for nu, p in STUDENT_QUANTILE: + print(f"student_quantile,{nu!r},{p!r},{fmt(student_quantile(nu, mpmath.mpf(p)))}") if __name__ == "__main__": diff --git a/crates/libitofin/tests/incomplete_beta_power_terms.rs b/crates/libitofin/tests/incomplete_beta_power_terms.rs index c17e0802..39e11d74 100644 --- a/crates/libitofin/tests/incomplete_beta_power_terms.rs +++ b/crates/libitofin/tests/incomplete_beta_power_terms.rs @@ -3,8 +3,17 @@ //! at 60 digits (`fixtures/ibeta_power_terms/generate.py`). use libitofin::math::distributions::binomial::CumulativeBinomialDistribution; +use libitofin::math::distributions::studentt::StudentT; +use libitofin::math::distributions::{Cdf, Probability, Quantile}; use libitofin::types::Real; +/// The forward form at `ν = 1e6` and the normal cdf's `erf` at `t = -5` set +/// this; the centre rows are within `1e-14`. +const STUDENT_REL_TOL: Real = 1e-10; + +/// The quantile is one Halley step from the normal estimate at these `ν`. +const QUANTILE_REL_TOL: Real = 1e-12; + /// Per-`n` tolerance of the Binomial rows: the continued fraction cancels near /// the saddle at these sizes (a documented deferral in `math/beta.rs`). fn binomial_tol(n: u64) -> Real { @@ -42,3 +51,44 @@ fn binomial_cdf_at_huge_n_matches_mpmath() { assert_rel(got, expected, binomial_tol(n), &context); } } + +/// The forward form (`|t| > sqrt(3)` here) passes the exact +/// `y = t²/(ν + t²)`, where `1 - z` rounds away the tail, and beyond +/// `ν = 2^52` the cdf is the normal one. +#[test] +fn student_cdf_at_huge_nu_matches_mpmath() { + let rows = rows("student,"); + assert_eq!(rows.len(), 32); + for row in rows { + let (nu, t, expected) = (row[0], row[1], row[2]); + let dist = StudentT::new(nu).unwrap(); + let context = format!("cdf nu={nu} t={t}"); + assert_rel(dist.cdf(t), expected, STUDENT_REL_TOL, &context); + } +} + +#[test] +fn student_quantile_at_huge_nu_matches_mpmath() { + let rows = rows("student_quantile,"); + assert_eq!(rows.len(), 2); + for row in rows { + let (nu, p, expected) = (row[0], row[1], row[2]); + let dist = StudentT::new(nu).unwrap(); + let got = dist.quantile(Probability::try_from(p).unwrap()).unwrap(); + assert_rel( + got, + expected, + QUANTILE_REL_TOL, + &format!("quantile nu={nu} p={p}"), + ); + } +} + +#[test] +fn student_cdf_normal_limit_keeps_the_endpoints() { + let dist = StudentT::new(1e16).unwrap(); + assert_eq!(dist.cdf(Real::NEG_INFINITY), 0.0); + assert_eq!(dist.cdf(Real::INFINITY), 1.0); + assert_eq!(dist.cdf(0.0), 0.5); + assert!(dist.cdf(Real::NAN).is_nan()); +} diff --git a/wiki/compatibility.md b/wiki/compatibility.md index 8fe58a9f..af060b29 100644 --- a/wiki/compatibility.md +++ b/wiki/compatibility.md @@ -134,6 +134,13 @@ oversight) and is documented at the point of divergence in the source. where `t²` overflows, the forward form is unchanged. Boost's own switch, `ν > 2t²` with the complemented `ibetac`, is not used: without a complemented incomplete beta it cancels the large-`ν` tails to 0. +- **The Student-t cdf is the normal cdf beyond `ν = 1/eps`.** QuantLib always + takes the incomplete beta; here, as in Boost's `students_t` cdf, `ν > 2^52` + returns the normal cdf, and below it both forms pass their exact complement + (`y` or `z`) to the incomplete beta rather than `1 - z` (#1468). Deferred: + for `|t| > √3` the forward form's continued fraction cancels at large `ν` + (about `1e-10` relative at `ν = 1e7`, `3e-5` at `1e12`, `0.1` at `2^52`), + where Boost uses `beta_small_b_large_a_series`. ## Cash flows (EPIC-7) From c6918b4845b15b341b5019cf0a241bf5c80507f7 Mon Sep 17 00:00:00 2001 From: benbenbang Date: Sat, 14 Nov 2026 22:35:03 -0500 Subject: [PATCH 4/5] docs(math): clarify binomial cdf accuracy comments per #1468 The CDF comment now records that incomplete_beta's Boost power-terms prefactor holds at large n, while its continued fraction cancels near the mean for n beyond about 1e11 (5e-10 relative at n = 1e12, 5e-7 at 1e16), a deferred limitation tracked in math/beta.rs. The cdf/pmf test comment no longer attributes the accuracy to Lanczos log_gamma; it now refers to the Boost Lanczos power terms, matching the current implementation. Comments only, no behaviour change. ref #1468 --- crates/libitofin/src/math/distributions/binomial.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/libitofin/src/math/distributions/binomial.rs b/crates/libitofin/src/math/distributions/binomial.rs index 3c8f68fb..179018e2 100644 --- a/crates/libitofin/src/math/distributions/binomial.rs +++ b/crates/libitofin/src/math/distributions/binomial.rs @@ -110,7 +110,10 @@ impl CumulativeBinomialDistribution { // its only residual failure is non-convergence, which its sqrt-scaled // iteration limit avoids on every swept input (#1463, // tests/incomplete_beta_large_a.rs; empirical, not a proof), - // asserted away per the eval error-boundary convention. + // asserted away per the eval error-boundary convention. Its Boost + // power-terms prefactor holds at large n (#1468), but near the mean + // the continued fraction cancels for n beyond about 1e11 (5e-10 + // relative at n = 1e12, 5e-7 at 1e16; deferred, see math/beta.rs). 1.0 - incomplete_beta((k + 1) as Real, (self.n - k) as Real, self.p) .expect("incomplete_beta converges for valid binomial CDF parameters") } @@ -178,8 +181,8 @@ mod tests { #[test] fn cdf_equals_cumulative_pmf() { - // The CDF goes through incomplete_beta (Lanczos log_gamma inside, ~1e-12 - // accurate), while the running PMF sum uses exact log-factorials, so they + // The CDF goes through incomplete_beta (Boost Lanczos power terms + // inside), while the running PMF sum uses exact log-factorials, so they // agree only to incomplete_beta's tolerance, not to ~1e-15. for &(p, n) in &[(0.2, 10u64), (0.5, 15), (0.75, 20)] { let pmf = BinomialDistribution::new(p, n).unwrap(); From 062b2b377acf7c04660072d3b6f69782957ee5ec Mon Sep 17 00:00:00 2001 From: benbenbang Date: Sat, 14 Nov 2026 22:46:59 -0500 Subject: [PATCH 5/5] fix(beta): assert y = 1 - x in incomplete_beta_xy `incomplete_beta_xy` documents that its caller must pass `y = 1 - x` but never verified it, so a caller that drifted would compute a wrong result silently. Add a `debug_assert` that `x + y` is within `8 eps` of `1`, skipped when `x` or `y` is non-finite so the Student-t endpoints still pass through. Every caller derives `x` and `y` from the same `t` and `nu`, so `8 eps` covers their rounding; all beta, Student-t and Binomial tests pass with it on in the debug build. Also cite the follow-up issues in the deferral notes: #1475 for the large-`a`,`b` asymptotic for the Binomial near the saddle (beta.rs and the wiki), and #1476 for `beta_small_b_large_a_series` on the Student-t forward tail plus the pdf normalization at huge `nu` (studentt.rs and the wiki). Refs: #1468, #1475, #1476 ref #1468 --- crates/libitofin/src/math/beta.rs | 6 +++++- crates/libitofin/src/math/distributions/studentt.rs | 2 +- wiki/compatibility.md | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/libitofin/src/math/beta.rs b/crates/libitofin/src/math/beta.rs index 8a81d6ea..538bfb83 100644 --- a/crates/libitofin/src/math/beta.rs +++ b/crates/libitofin/src/math/beta.rs @@ -116,7 +116,7 @@ pub fn beta_function(z: Real, w: Real) -> QlResult { /// cancels for `a`, `b` beyond about `1e11`, so the Binomial cdf at /// `n = 1e12, 1e14, 1e16` is within `5e-10`, `1e-8` and `5e-7` of mpmath /// there, not `1e-10`; a large-parameter asymptotic expansion is tracked -/// separately. +/// as #1475. /// /// # Errors /// @@ -148,6 +148,10 @@ pub fn incomplete_beta(a: Real, b: Real, x: Real) -> QlResult { /// /// As [`incomplete_beta`], and also if `y` is outside `[0, 1]`. pub(crate) fn incomplete_beta_xy(a: Real, b: Real, x: Real, y: Real) -> QlResult { + debug_assert!( + !(x.is_finite() && y.is_finite()) || (x + y - 1.0).abs() <= 8.0 * Real::EPSILON, + "incomplete_beta_xy needs y = 1 - x (x = {x}, y = {y})" + ); // `is_finite()` is required: a bare `<= 0.0` lets NaN and +infinity through // (all NaN comparisons are false, and +infinity is not <= 0), but // QuantLib's QL_REQUIRE(a > 0) rejects them diff --git a/crates/libitofin/src/math/distributions/studentt.rs b/crates/libitofin/src/math/distributions/studentt.rs index f6177b35..9e3ac5c8 100644 --- a/crates/libitofin/src/math/distributions/studentt.rs +++ b/crates/libitofin/src/math/distributions/studentt.rs @@ -105,7 +105,7 @@ impl Cdf for StudentT { // form's continued fraction cancels at z near 1 for large ν (|x| > √3: // about 1e-10 relative at ν = 1e7, 3e-5 at 1e12, 0.1 at 2^52), where // Boost switches to beta_small_b_large_a_series; that port is tracked - // separately. + // as #1476. const CONVERGES: &str = "incomplete beta must converge for valid Student-t parameters"; if y * (nu + 5.0) < 3.0 { let half = 0.5 * incomplete_beta_xy(0.5, 0.5 * nu, y, z).expect(CONVERGES); diff --git a/wiki/compatibility.md b/wiki/compatibility.md index af060b29..8a052a25 100644 --- a/wiki/compatibility.md +++ b/wiki/compatibility.md @@ -122,7 +122,7 @@ oversight) and is documented at the point of divergence in the source. still cancels beyond about `1e11`: the Binomial cdf at `p = 1/2` and `n = 1e12, 1e14, 1e16` is within `5e-10`, `1e-8`, `5e-7` of mpmath (Boost 1.92 itself is within `7e-11`, `3e-10`, `8e-9`), deferred to a - large-parameter asymptotic. + large-parameter asymptotic (#1475). - **The Student-t cdf evaluates the centre directly.** QuantLib's `CumulativeStudentDistribution` (`studenttdistribution.cpp`) always forms `I_x(ν/2, 1/2) / 2` with `x = ν/(ν + t²)`; near `t = 0` that `x` rounds @@ -140,7 +140,7 @@ oversight) and is documented at the point of divergence in the source. (`y` or `z`) to the incomplete beta rather than `1 - z` (#1468). Deferred: for `|t| > √3` the forward form's continued fraction cancels at large `ν` (about `1e-10` relative at `ν = 1e7`, `3e-5` at `1e12`, `0.1` at `2^52`), - where Boost uses `beta_small_b_large_a_series`. + where Boost uses `beta_small_b_large_a_series` (#1476). ## Cash flows (EPIC-7)