diff --git a/ALPS3_MIGRATION_PLAN.md b/ALPS3_MIGRATION_PLAN.md index f2a81fe..d36dd4d 100644 --- a/ALPS3_MIGRATION_PLAN.md +++ b/ALPS3_MIGRATION_PLAN.md @@ -223,9 +223,11 @@ Measured facts that the design relies on: | Targeted cases | Covariance (text, HDF5); `DATA_IN_HDF5`; `X_i`/`SIGMA_i` input; explicit `TAU_i`; T=0, time-bosonic, anomalous (PH and non-PH) kernels; every default model incl. tabulated; every grid; `MODEL_RUNS` (varspec) | same, plus a component harness that dumps the kernel matrix, its singular values, the grid and the discretized default model for small sizes | | CLI snapshots | `--help`, `--help.models`, `--help.grids`, missing-parameter error | exact text (exit codes once B1 is fixed) | +`GENERATE_ERR` is covered by `t_generate_err`; its bootstrap output is +reproducible because `SEED` defaults to 0 (B10). + Not captured: -* `GENERATE_ERR`: seeded from the clock (B10). It gets a reference after the seed parameter is added. * The LAPACK SVD path: compared against the Eigen results within tolerance, with no separate reference. Cases that run through suspected bugs record current behavior and are flagged @@ -406,20 +408,29 @@ need careful checking at large l and argument) and `program_options` (no standard equivalent). Reconfirmed 2026-09-24: these are intentional retained Boost boundaries for now. -**2.3C: command-line behavior** (updates the CLI references on purpose) - -* B1: non-zero exit code on errors, together with `boost::diagnostic_information` - → `e.what()` (moved here from 2.2). -* B16: accept `half-lorentzian` as well as `half lorentzian`, and show the real - `CUT` default in `--help.grids`. -* B10: a `SEED` parameter for the bootstrap errors (moved here from 2.2). - -**2.3D: numerical changes** (each its own commit and `REFERENCE_CHANGES.md` entry) - -* B7b: time-bosonic kernel: use the ω→0 limit only at ω = 0. -* B15: remove the `float` casts in the log grid. -* B11 (optional): `BDCSVD` instead of `JacobiSVD`; measure against the - references before deciding. +**2.3C: command-line behavior — Done 2026-09-24** (updates the CLI references on purpose) + +* **Done 2026-09-24:** B1: non-zero exit code on errors, together with + `boost::diagnostic_information` → `e.what()` (moved here from 2.2). +* **Done 2026-09-24:** B16: accept `half-lorentzian` as well as + `half lorentzian`, and show the real `CUT` default in `--help.grids`. +* **Done 2026-09-24:** B10: a `SEED` parameter for the bootstrap errors, + defaulting to 0 for reproducibility (moved here from 2.2). Added the + `t_generate_err` regression reference for the deterministic default and + `t_generate_err_seed` to verify that an explicit non-default seed is used. + +**2.3D: numerical changes — Done 2026-09-24** (each accepted change its own +commit and `REFERENCE_CHANGES.md` entry) + +* **Done 2026-09-24:** B7b: time-bosonic kernel uses the ω→0 limit only + at ω = 0. Updated the targeted and component references. +* **Done 2026-09-24:** B15: replaced the log-grid `float` casts with explicit + `double` conversions. Existing grids are bit-identical (the integer values + used by the tests and normal runs are exactly representable as `float`). +* **Measured 2026-09-24; keep `JacobiSVD`:** `BDCSVD` did not improve the + large Legendre case (57.49 s, effectively unchanged), while its different + singular vectors changed the deterministic bootstrap-error result by about + 3%. The optional B11 change therefore has no demonstrated benefit here. Keep the `MaxEntSimulation` public getters stable, because the tests use them. diff --git a/src/maxent.cpp b/src/maxent.cpp index 8d4639a..5fdba78 100644 --- a/src/maxent.cpp +++ b/src/maxent.cpp @@ -11,11 +11,11 @@ #include "maxent.hpp" #include -#include int main(int argc,const char** argv) { + try { alps::params parms(argc,argv); MaxEntSimulation::define_parameters(parms); //other help messages @@ -62,8 +62,8 @@ int main(int argc,const char** argv) std::cout << "For more information see examples/grids.pdf\n" << std::endl; std::cout <(); - try{ if(exitEarly){ throw std::runtime_error("Critical parameters not defined"); } @@ -163,11 +162,14 @@ int main(int argc,const char** argv) my_sim.run(); my_sim.evaluate(); } + return 0; } catch(const std::exception &e){ - std::cerr << "Caught Exception " << boost::diagnostic_information(e); + std::cerr << "Caught Exception: " << e.what() << '\n'; + return 1; } catch(...){ - std::cerr << "Caught Exception" << boost::current_exception_diagnostic_information(); + std::cerr << "Caught Exception: unknown exception\n"; + return 1; } } diff --git a/src/maxent.hpp b/src/maxent.hpp index fb4bf39..1df6877 100644 --- a/src/maxent.hpp +++ b/src/maxent.hpp @@ -70,6 +70,7 @@ public : void generateCovariantErr(const vector_type& A_in, const double alpha, ofstream_ &os); private: + std::mt19937::result_type bootstrap_seed_; ///discretized and normalized version of the default model. vector_type def_; ///check that the default model is non-zero diff --git a/src/maxent_grid.cpp b/src/maxent_grid.cpp index 30b96b5..5f7f973 100644 --- a/src/maxent_grid.cpp +++ b/src/maxent_grid.cpp @@ -22,7 +22,7 @@ t_array_(nfreq_+1){ if (p_f_grid =="lorentzian") { initialize_lorentzian_grid(cut); } - else if (p_f_grid=="half lorentzian") { + else if (p_f_grid=="half lorentzian" || p_f_grid=="half-lorentzian") { initialize_half_lorentzian_grid(cut); } else if (p_f_grid=="quadratic") { @@ -46,18 +46,18 @@ void grid::initialize_linear_grid() { void grid::initialize_logarithmic_grid(double t_min) { double t_max = 0.5; - double scale = std::log(t_max / t_min) / ((float) ((nfreq_ / 2 - 1))); + double scale = std::log(t_max / t_min) / static_cast(nfreq_ / 2 - 1); t_array_[nfreq_ / 2] = 0.5; for (int i = 0; i < nfreq_ / 2; ++i) { t_array_[nfreq_ / 2 + i + 1] = 0.5 - + t_min * std::exp(((float) (i)) * scale); + + t_min * std::exp(static_cast(i) * scale); t_array_[nfreq_ / 2 - i - 1] = 0.5 - - t_min * std::exp(((float) (i)) * scale); + - t_min * std::exp(static_cast(i) * scale); } //if we have an odd # of frequencies, this catches the last element if (nfreq_ % 2 != 0) t_array_[nfreq_ / 2 + nfreq_ / 2 + 1] = 0.5 - + t_min * std::exp(((float) (nfreq_) / 2) * scale); + + t_min * std::exp(static_cast(nfreq_) / 2 * scale); } void grid::initialize_quadratic_grid(double spread) { @@ -100,5 +100,3 @@ void grid::initialize_lorentzian_grid(double cut) { } - - diff --git a/src/maxent_helper.cpp b/src/maxent_helper.cpp index 78f2624..406a8ad 100644 --- a/src/maxent_helper.cpp +++ b/src/maxent_helper.cpp @@ -11,7 +11,6 @@ #include "maxent.hpp" #include -#include #include #include #include "maxent_backcont.hpp" @@ -21,7 +20,9 @@ //NOTE: size1= rows; size2=columns MaxEntHelper::MaxEntHelper(alps::params& p) : -MaxEntParameters(p) , def_(nfreq()), text_output(p["TEXT_OUTPUT"]) +MaxEntParameters(p), +bootstrap_seed_(p["SEED"].as()), +def_(nfreq()), text_output(p["TEXT_OUTPUT"]) { for (int i=0; i(std::time(0))); + std::mt19937 rng(bootstrap_seed_); std::vector noise_vecs; int max_it = 10000; diff --git a/src/maxent_kernel.cpp b/src/maxent_kernel.cpp index b5ec5c6..3e782c2 100644 --- a/src/maxent_kernel.cpp +++ b/src/maxent_kernel.cpp @@ -79,10 +79,10 @@ K_(ndat_,nfreq_) else if (ktype_==time_bosonic_kernel) { for (int i=0; i -Infity } } diff --git a/src/maxent_simulation.cpp b/src/maxent_simulation.cpp index 5282782..0e761c5 100644 --- a/src/maxent_simulation.cpp +++ b/src/maxent_simulation.cpp @@ -56,6 +56,7 @@ void MaxEntSimulation::define_parameters(alps::params &p){ p.define("NORM",1.0,"NORM"); p.define("BACKCONTINUE",true,"Output A(omega) back to imaginary axis"); p.define("GENERATE_ERR",false,"Generate a bootstrap approximation for error bars"); + p.define("SEED",0,"Seed for reproducible bootstrap error bars"); //********************************* p.define("BETA","beta, inverse temperature"); p.define("NDAT","# of input points"); diff --git a/test/gridTest.cpp b/test/gridTest.cpp index 16d3563..c0815cf 100644 --- a/test/gridTest.cpp +++ b/test/gridTest.cpp @@ -104,7 +104,21 @@ TEST(Grid,HalfLorentzianEven){ grid g(p); EXPECT_EQ(g(0),0); - EXPECT_EQ(g(NFREQ),1); + EXPECT_EQ(g(NFREQ),1); +} +TEST(Grid,HalfLorentzianHyphenAlias){ + alps::params p; + MaxEntSimulation::define_parameters(p); + const int NFREQ=20; + p["NFREQ"] = NFREQ; + + p["FREQUENCY_GRID"] = "half lorentzian"; + grid spaced(p); + p["FREQUENCY_GRID"] = "half-lorentzian"; + grid hyphenated(p); + + for(int i=0;i<=NFREQ;i++) + EXPECT_EQ(hyphenated(i),spaced(i)); } TEST(Grid,QuadraticEven){ alps::params p; @@ -147,4 +161,3 @@ TEST(Grid,LinearEven){ } - diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index 97e109b..8882557 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -27,6 +27,10 @@ if(NOT have_python_modules) return() endif() +add_test(NAME regression_harness_unit + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_generate.py) +set_tests_properties(regression_harness_unit PROPERTIES LABELS unit) + function(maxent_regression_test name sets label) set(results ${CMAKE_CURRENT_BINARY_DIR}/results-${name}) add_test(NAME regression_${name}_generate diff --git a/test/regression/MANIFEST.md b/test/regression/MANIFEST.md index 5a3fd03..14a139e 100644 --- a/test/regression/MANIFEST.md +++ b/test/regression/MANIFEST.md @@ -24,7 +24,9 @@ Generated by `manifest.py` from `reference/`, `cases.py` and `tolerances.json`. |---|---|---|---|---|---| | `t_cov_hdf5` | DATA_IN_HDF5 with /Covariance, frequency PH | `test/regression/inputs/t_cov_hdf5` | 20 | 0.1 s | | | `t_cov_text` | covariance matrix from text file, time fermionic | `test/regression/inputs/t_cov_text` | 20 | 0.1 s | | -| `t_grid_half_lorentzian` | grid: half lorentzian | `test/regression/inputs/t_grid_half_lorentzian` | 20 | 0.1 s | B16 (help text says 'half-lorentzian') | +| `t_generate_err` | bootstrap error bars with the default SEED=0 | `test/regression/inputs/t_generate_err` | 21 | 0.0 s | | +| `t_generate_err_seed` | bootstrap error bars with explicit SEED=1234 | `test/regression/inputs/t_generate_err_seed` | 21 | 0.1 s | | +| `t_grid_half_lorentzian` | grid: half-lorentzian alias | `test/regression/inputs/t_grid_half_lorentzian` | 20 | 0.1 s | | | `t_grid_linear` | grid: linear | `test/regression/inputs/t_grid_linear` | 20 | 0.1 s | | | `t_grid_log` | grid: log | `test/regression/inputs/t_grid_log` | 20 | 0.1 s | | | `t_grid_lorentzian` | grid: lorentzian | `test/regression/inputs/t_grid_lorentzian` | 20 | 0.1 s | | @@ -33,7 +35,7 @@ Generated by `manifest.py` from `reference/`, `cases.py` and `tolerances.json`. | `t_kernel_anomalous_nonph` | anomalous kernel, frequency non-PH | `test/regression/inputs/t_kernel_anomalous_nonph` | 24 | 0.1 s | | | `t_kernel_anomalous_ph` | anomalous kernel, frequency PH | `test/regression/inputs/t_kernel_anomalous_ph` | 24 | 0.1 s | | | `t_kernel_legendre_bosonic` | Legendre bosonic kernel | `test/regression/inputs/t_kernel_legendre_bosonic` | 24 | 0.1 s | B17 (Legendre bosonic uses the fermionic integrand) | -| `t_kernel_time_bosonic` | time bosonic kernel, OMEGA_MIN=0 | `test/regression/inputs/t_kernel_time_bosonic` | 24 | 0.1 s | B7 (fixed; K(i,0)=T assumes the grid starts at 0) | +| `t_kernel_time_bosonic` | time bosonic kernel, OMEGA_MIN=0 | `test/regression/inputs/t_kernel_time_bosonic` | 24 | 0.3 s | | | `t_kernel_tzero` | T=0 kernel, OMEGA_MIN=0 | `test/regression/inputs/t_kernel_tzero` | 20 | 0.1 s | | | `t_model_double_gaussian` | default model: double gaussian | `test/regression/inputs/t_model_double_gaussian` | 20 | 0.1 s | | | `t_model_double_lorentzian` | default model: double lorentzian | `test/regression/inputs/t_model_double_lorentzian` | 20 | 0.1 s | | @@ -57,9 +59,9 @@ Generated by `manifest.py` from `reference/`, `cases.py` and `tolerances.json`. | Case | Covers | Inputs | Datasets | Runtime | Flags | |---|---|---|---|---|---| | `cli_help` | --help output (`--help`) | none | 8 | 0.0 s | | -| `cli_help_grids` | --help.grids output (`--help.grids`) | none | 8 | 0.0 s | B16 | +| `cli_help_grids` | --help.grids output (`--help.grids`) | none | 8 | 0.0 s | | | `cli_help_models` | --help.models output (`--help.models`) | none | 8 | 0.0 s | | -| `cli_missing_beta` | error output when BETA is missing (B1: exit code is 0) | `test/regression/inputs/cli_missing_beta` | 8 | 0.0 s | B1; expects exception | +| `cli_missing_beta` | error output and failing exit status when BETA is missing | `test/regression/inputs/cli_missing_beta` | 8 | 0.4 s | expects exception | ## Full set (examples as shipped, opt-in) @@ -95,7 +97,7 @@ Generated by `manifest.py` from `reference/`, `cases.py` and `tolerances.json`. | Case | Covers | Inputs | Datasets | Runtime | Flags | |---|---|---|---|---|---| -| `components` | kernel matrices (11 reachable kernel types), grids (5 types, odd/even), default models (13), singular values | `components/dump_components.cpp` | 47 | 0.0 s | | +| `components` | kernel matrices (11 reachable kernel types), grids (5 types, odd/even), default models (13), singular values | `components/dump_components.cpp` | 49 | 0.4 s | | ## Tolerances @@ -119,10 +121,10 @@ Rules are matched in order against ':'; the first match applies. | `\.chi2\.dat$` | 0 | 1e-07 | observed 1.7e-9 | | `:files/` | 1e-15 | 2e-06 | spectra, back-continuations, grids, all other outputs; observed 1.5e-7 (GCC 15, u2_time_fast) | -Total compared datasets: 1192 in 60 cases. +Total compared datasets: 1236 in 62 cases. ## Known limitations * The Legendre example is extremely ill-conditioned (minimal chi2 ~ 1e-17): rounding-level differences in the kernel change its alpha probabilities and Ng by up to 3e-3 and 3e-2 between compilers, so its end-to-end tolerances are loose. Kernel changes are caught by the Legendre kernel matrices in `components.h5`, compared at 1e-14. (Checks: Boost Gauss-Kronrod quadrature and, since step 2.2, the closed form used in maxent_kernel.cpp reproduce the GSL Legendre kernels of the references to 2e-16 and 3.3e-16.) * `-O0` builds take more than 30 minutes for the full `legendre` case. -* `*.spex.dat`, `*.fits.dat` and `*.booterr.dat` are not compared (see README.md). +* `*.spex.dat` and `*.fits.dat` are not compared (see README.md). diff --git a/test/regression/PROVENANCE.md b/test/regression/PROVENANCE.md index f7de757..8f3e1b7 100644 --- a/test/regression/PROVENANCE.md +++ b/test/regression/PROVENANCE.md @@ -1,9 +1,10 @@ # Provenance of the references -The references in `reference/` were generated on 2026-09-23 from the +Most references in `reference/` were generated on 2026-09-23 from the **original numerics** of Maxent, before any modernization of the code. -The same information is stored as `provenance.*` attributes at the root of -every reference file. +References added or deliberately regenerated later are identified in the +dated sections below and in `REFERENCE_CHANGES.md`. Corresponding provenance +is stored as `provenance.*` attributes at each reference root. ## Source @@ -54,11 +55,33 @@ Its synthetic Gaussian-spectrum input has zero error bars, so the clock-seeded bootstrap is deterministic; three consecutive generate-and-compare runs passed. +## Bootstrap-error reference (added 2026-09-24) + +`t_generate_err.h5` was generated from `modernize/step2.3c` after commit +`379b2f0`, using the Release build with AppleClang 21, libc++, C++17, and the +local ALPSCore installation. It uses the new default `SEED=0`; two independent +generations were identical. `t_generate_err_seed.h5` was added after commit +`8b2ff05` with explicit `SEED=1234`; its bootstrap values differ from the +default-seed reference, so the case detects an ignored `SEED` parameter. Both +references include `case.out.booterr.dat`. + +## Time-bosonic references updated (B7b, 2026-09-24) + +The `kernel_time_bosonic` datasets in `components.h5` and the +`t_kernel_time_bosonic.h5` reference were regenerated from +`modernize/step2.3c` after commit `46b5610`, using the Release build with +AppleClang 21 and libc++. All other component datasets are unchanged. +The component reference also adds a symmetric-grid time-bosonic kernel to +cover a negative first frequency and the exact ω=0 branch. + ## Determinism -Two independent generations with the reference build were compared with -`compare.py --report`: all 1165 datasets (1157 numeric, 8 text) in 56 cases -were bit-identical. +For the initial 2026-09-23 baseline, two independent generations were compared +with `compare.py --report`: all 1165 datasets (1157 numeric, 8 text) in the +original 56 cases were bit-identical. After the subsequently documented +reference additions and updates, the current inventory is 1236 compared +datasets (1228 numeric, 8 text) in 62 cases; see `MANIFEST.md` for the generated +current count. ## Variant builds (tolerance measurement) diff --git a/test/regression/README.md b/test/regression/README.md index eb1dc3e..dbc9bbe 100644 --- a/test/regression/README.md +++ b/test/regression/README.md @@ -73,8 +73,10 @@ Each result file holds: copied dataset by dataset at full precision. Text tables (`*.dat`) are loaded as float64, but only carry the 10 significant digits Maxent writes. `*.spex.dat` and `*.fits.dat` (spectrum and fit at every α; large, and - covered by `chi2.dat`, the α probabilities and the `*_back.dat` files) and - `*.booterr.dat` (random, B10) are not packed. + covered by `chi2.dat`, the α probabilities and the `*_back.dat` files) are + not packed. Bootstrap error output is deterministic (`SEED=0` by default) + and is covered by `t_generate_err`; `t_generate_err_seed` verifies an + explicit non-default seed. * `/log/scalars/`: values parsed from the logs: minimal χ², posterior probability of the default model, Ng, number of singular values kept, and the number of α values where the minimizer hit `MAX_IT`. diff --git a/test/regression/REFERENCE_CHANGES.md b/test/regression/REFERENCE_CHANGES.md index 503ff6d..ebb3feb 100644 --- a/test/regression/REFERENCE_CHANGES.md +++ b/test/regression/REFERENCE_CHANGES.md @@ -4,6 +4,64 @@ Every regeneration of files in `reference/` is logged here: date, commit, reason, affected cases, and the largest difference to the previous references (from `compare.py --report`). Newest entry first. +## 2026-09-24: explicit bootstrap seed coverage + +* **Source:** `modernize/step2.3c` after commit `8b2ff05`. +* **Reason:** added review-requested coverage proving that a non-default + `SEED` value is parsed and used, rather than only testing the default. +* **Cases:** added `t_generate_err_seed` with `SEED=1234`, including + `booterr.dat`. It uses the exact input of the default-seed case, isolating + the seed as the only cause of bootstrap differences. No existing reference + changed. + +## 2026-09-24: log-grid float casts removed (B15) + +* **Source:** `modernize/step2.3c` after commit `b1d7c8b`. +* **Reason:** log-grid intermediates now remain explicitly in double + precision instead of passing integer indices through `float`. +* **Cases:** `t_grid_log` and both log-grid component sizes were measured. + They are bit-identical, so no reference was regenerated. + +## 2026-09-24: time-bosonic zero-frequency limit corrected (B7b) + +* **Source:** `modernize/step2.3c` after commit `46b5610`. +* **Reason:** the time-bosonic kernel previously assigned the ω→0 limit to + column 0 regardless of that column's frequency. It now uses the limit only + when ω is exactly zero and evaluates the kernel formula everywhere else. +* **Cases:** `t_kernel_time_bosonic` and `components`; added a symmetric-grid + time-bosonic component that contains both negative and exactly zero ω. +* **Largest differences:** 2.202e-3 relative in the kernel component; 5.373e-4 + relative in χ² and 1.190e-2 relative in the small spectrum variance. + +## 2026-09-24: reproducible bootstrap errors (B10) + +* **Source:** `modernize/step2.3c` after commit `379b2f0`. +* **Reason:** bootstrap error generation now uses the user-selectable `SEED` + parameter, which defaults to 0 instead of the wall clock. +* **Cases:** added `t_generate_err`, including `booterr.dat`; updated + `cli_help` for the new parameter. No existing numerical reference changed. +* **Verification:** two independent runs with the default seed were identical. + +## 2026-09-24: grid spelling and help corrected (B16) + +* **Source:** `modernize/step2.3c`, based on merge commit `454f7ee`. +* **Reason:** expected help-text correction: `--help.grids` now reports the + actual `CUT=0.01` default. The documented `half-lorentzian` spelling is now + accepted as an alias for `half lorentzian`. +* **Cases:** `cli_help_grids` text only. The targeted half-Lorentzian case now + uses the hyphenated alias and remains within its existing numerical + tolerance; no numerical reference was regenerated. + +## 2026-09-24: command-line failures return nonzero (B1) + +* **Source:** `modernize/step2.3c`, based on merge commit `454f7ee`. +* **Reason:** expected behavior change: Maxent now returns status 1 on errors + and reports standard exceptions with `e.what()` instead of Boost diagnostic + formatting. +* **Cases:** `cli_missing_beta` only. Exit status changed from 0 to 1; stderr + changed from the multi-line Boost diagnostic to + `Caught Exception: Critical parameters not defined`. + ## 2026-09-24: legendre_convert reference added * **Source:** `modernize/step2` merge commit `a57f150`; the utility is still diff --git a/test/regression/cases.py b/test/regression/cases.py index 43b89b2..8e21c54 100644 --- a/test/regression/cases.py +++ b/test/regression/cases.py @@ -44,7 +44,7 @@ ("t_param_xi", "data as X_i/SIGMA_i in parameter file, frequency PH", []), ("t_param_taui", "X_i/SIGMA_i/TAU_i in parameter file, time fermionic", []), ("t_kernel_tzero", "T=0 kernel, OMEGA_MIN=0", []), - ("t_kernel_time_bosonic", "time bosonic kernel, OMEGA_MIN=0", ["B7 (fixed; K(i,0)=T assumes the grid starts at 0)"]), + ("t_kernel_time_bosonic", "time bosonic kernel, OMEGA_MIN=0", []), ("t_kernel_anomalous_ph", "anomalous kernel, frequency PH", []), ("t_kernel_anomalous_nonph", "anomalous kernel, frequency non-PH", []), ("t_kernel_legendre_bosonic", "Legendre bosonic kernel", ["B17 (Legendre bosonic uses the fermionic integrand)"]), @@ -62,10 +62,12 @@ ("t_model_quadratic_rise_exp_decay", "default model: quadratic rise exp decay (T=0 kernel, OMEGA_MIN=0)", []), ("t_model_tabulated", "default model: tabulated file", []), ("t_grid_lorentzian", "grid: lorentzian", []), - ("t_grid_half_lorentzian", "grid: half lorentzian", ["B16 (help text says 'half-lorentzian')"]), + ("t_grid_half_lorentzian", "grid: half-lorentzian alias", []), ("t_grid_quadratic", "grid: quadratic", []), ("t_grid_log", "grid: log", []), ("t_grid_linear", "grid: linear", []), + ("t_generate_err", "bootstrap error bars with the default SEED=0", []), + ("t_generate_err_seed", "bootstrap error bars with explicit SEED=1234", []), ("t_model_runs", "MODEL_RUNS=2 (flat, gaussian): per-model outputs and varspec", []), ] @@ -73,7 +75,7 @@ ("cli_help", ["--help"], "--help output"), ("cli_help_models", ["--help.models"], "--help.models output"), ("cli_help_grids", ["--help.grids"], "--help.grids output"), - ("cli_missing_beta", None, "error output when BETA is missing (B1: exit code is 0)"), + ("cli_missing_beta", None, "error output and failing exit status when BETA is missing"), ] @@ -115,12 +117,12 @@ def all_cases(): for name, args, covers in CLI: if args is None: cases.append(dict(name=name, set="cli", inputs="test/regression/inputs/" + name, - param="case.param", args=[], covers=covers, flags=["B1"], + param="case.param", args=[], covers=covers, flags=[], expect="exception")) else: cases.append(dict(name=name, set="cli", inputs=None, param=None, args=args, covers=covers, - flags=["B16"] if name == "cli_help_grids" else [], + flags=[], expect="ok")) for name, direction, covers in KK: cases.append(dict(name=name, set="kk", program="kk", inputs="test/regression/inputs/" + name, diff --git a/test/regression/components/dump_components.cpp b/test/regression/components/dump_components.cpp index b1fb3ef..55d243b 100644 --- a/test/regression/components/dump_components.cpp +++ b/test/regression/components/dump_components.cpp @@ -58,9 +58,9 @@ alps::params base_params() { // ---------------------------------------------------------------- kernels void dump_kernel(const std::string &name, const std::string &dataspace, const std::string &kernel_name, - bool ph, int ndat, double omega_min, double omega_max, bool time_input) { + bool ph, int ndat, double omega_min, double omega_max, bool time_input, + int nfreq = 24) { alps::params p = base_params(); - const int nfreq = 24; p["NDAT"] = ndat; p["NFREQ"] = nfreq; p["DATASPACE"] = dataspace; @@ -137,6 +137,7 @@ int main(int argc, char **argv) { // kernels: every kernel type reachable through set_kernel_type (the time-Legendre types are not, B5) dump_kernel("time_fermionic", "time", "fermionic", false, 9, -6, 6, true); dump_kernel("time_bosonic", "time", "bosonic", false, 9, 0, 6, true); + dump_kernel("time_bosonic_symmetric", "time", "bosonic", false, 9, -6, 6, true, 25); dump_kernel("time_tzero", "time", "tzero", false, 9, 0, 6, true); dump_kernel("legendre_fermionic", "legendre", "fermionic", true, 6, -6, 6, false); dump_kernel("legendre_bosonic", "legendre", "bosonic", true, 6, -6, 6, false); diff --git a/test/regression/generate.py b/test/regression/generate.py index 8d5007c..f59208c 100644 --- a/test/regression/generate.py +++ b/test/regression/generate.py @@ -18,7 +18,7 @@ Not packed: *.spex.dat and *.fits.dat (spectrum and fit for every alpha: they grow like N_ALPHA x NFREQ or N_ALPHA x NDAT, and are covered by chi2.dat, the -alpha probabilities and the *_back.dat files) and *.booterr.dat (random, B10). +alpha probabilities and the *_back.dat files). """ import argparse @@ -40,7 +40,7 @@ sys.path.insert(0, str(HERE)) from cases import all_cases # noqa: E402 -SKIP_OUTPUTS = re.compile(r"\.(spex|fits|booterr)\.dat$") +SKIP_OUTPUTS = re.compile(r"\.(spex|fits)\.dat$") SCALARS = { "minimal_chi2": re.compile(r"^minimal chi2: (\S+)", re.M), "posterior_probability": re.compile(r"^posterior probability of the default model: (\S+)", re.M), @@ -62,6 +62,17 @@ def run(cmd, cwd, timeout=TIMEOUT): return p, time.monotonic() - t0 +def classify_status(program, returncode, stderr): + """Classify command completion without mistaking crashes for exceptions.""" + if returncode is None: + return "timeout" + if program == "maxent" and returncode == 1 and "Caught Exception:" in stderr: + return "exception" + if returncode != 0: + return "failed" + return "ok" + + def pack_outputs(h5, workdir, before): files = h5.create_group("files") for path in sorted(workdir.rglob("*")): @@ -114,11 +125,10 @@ def run_case(case, programs, outdir, provenance, timeout=TIMEOUT): before = {p.relative_to(workdir).as_posix() for p in workdir.rglob("*")} cmd = [str(programs[case["program"]])] + ([case["param"]] if case["param"] else []) + case["args"] p, seconds = run(cmd, workdir, timeout) - # maxent always exits with 0 (B1) and reports errors as 'Caught Exception'; - # the other programs report failure through their exit code - status = "timeout" if p.returncode is None else \ - "exception" if "Caught Exception" in p.stderr else \ - "failed" if case["program"] != "maxent" and p.returncode != 0 else "ok" + # An expected Maxent exception has both the documented exit status and + # diagnostic signature. Crashes and other nonzero exits remain generic + # failures rather than accidentally satisfying an exception case. + status = classify_status(case["program"], p.returncode, p.stderr) with h5py.File(out, "w") as h5: for key in ("name", "set", "covers"): h5.attrs[key] = case[key] diff --git a/test/regression/inputs/t_generate_err/case.param b/test/regression/inputs/t_generate_err/case.param new file mode 100644 index 0000000..e8c3cf9 --- /dev/null +++ b/test/regression/inputs/t_generate_err/case.param @@ -0,0 +1,9 @@ +BETA=5.0 +NFREQ=40 +N_ALPHA=8 +NDAT=24 +DATASPACE=frequency +KERNEL=fermionic +PARTICLE_HOLE_SYMMETRY=true +DATA="data.dat" +GENERATE_ERR=true diff --git a/test/regression/inputs/t_generate_err/data.dat b/test/regression/inputs/t_generate_err/data.dat new file mode 100644 index 0000000..d74bcda --- /dev/null +++ b/test/regression/inputs/t_generate_err/data.dat @@ -0,0 +1,24 @@ +0.62831853071795862 -0.80874290991409159 0.0001 +1.8849555921538759 -0.44014716084961508 0.0001 +3.1415926535897931 -0.29286728018013575 0.0001 +4.3982297150257104 -0.21712083835923829 0.0001 +5.6548667764616276 -0.17165635623034461 0.0001 +6.911503837897544 -0.14166307358889488 0.0001 +8.1681408993334621 -0.12060505637657916 0.0001 +9.4247779607693793 -0.10485266703834568 0.0001 +10.681415022205297 -0.092950172398290121 0.0001 +11.938052083641214 -0.083168742498363424 0.0001 +13.194689145077131 -0.075316139276522379 0.0001 +14.451326206513048 -0.068980526475614631 0.0001 +15.707963267948966 -0.063426614004070431 0.0001 +16.964600329384883 -0.058669718153089162 0.0001 +18.2212373908208 -0.054868212835967246 0.0001 +19.477874452256717 -0.051194487824511997 0.0001 +20.734511513692635 -0.048183803227436109 0.0001 +21.991148575128552 -0.045353889185727227 0.0001 +23.247785636564469 -0.04311132391388374 0.0001 +24.504422698000386 -0.040812383784974739 0.0001 +25.7610597594363 -0.038789286218401768 0.0001 +27.017696820872221 -0.037114935461959209 0.0001 +28.274333882308138 -0.035393491249622627 0.0001 +29.530970943744059 -0.033879643568396858 0.0001 diff --git a/test/regression/inputs/t_generate_err_seed/case.param b/test/regression/inputs/t_generate_err_seed/case.param new file mode 100644 index 0000000..ad67256 --- /dev/null +++ b/test/regression/inputs/t_generate_err_seed/case.param @@ -0,0 +1,10 @@ +BETA=5.0 +NFREQ=40 +N_ALPHA=8 +NDAT=24 +DATASPACE=frequency +KERNEL=fermionic +PARTICLE_HOLE_SYMMETRY=true +DATA="data.dat" +GENERATE_ERR=true +SEED=1234 diff --git a/test/regression/inputs/t_generate_err_seed/data.dat b/test/regression/inputs/t_generate_err_seed/data.dat new file mode 100644 index 0000000..d74bcda --- /dev/null +++ b/test/regression/inputs/t_generate_err_seed/data.dat @@ -0,0 +1,24 @@ +0.62831853071795862 -0.80874290991409159 0.0001 +1.8849555921538759 -0.44014716084961508 0.0001 +3.1415926535897931 -0.29286728018013575 0.0001 +4.3982297150257104 -0.21712083835923829 0.0001 +5.6548667764616276 -0.17165635623034461 0.0001 +6.911503837897544 -0.14166307358889488 0.0001 +8.1681408993334621 -0.12060505637657916 0.0001 +9.4247779607693793 -0.10485266703834568 0.0001 +10.681415022205297 -0.092950172398290121 0.0001 +11.938052083641214 -0.083168742498363424 0.0001 +13.194689145077131 -0.075316139276522379 0.0001 +14.451326206513048 -0.068980526475614631 0.0001 +15.707963267948966 -0.063426614004070431 0.0001 +16.964600329384883 -0.058669718153089162 0.0001 +18.2212373908208 -0.054868212835967246 0.0001 +19.477874452256717 -0.051194487824511997 0.0001 +20.734511513692635 -0.048183803227436109 0.0001 +21.991148575128552 -0.045353889185727227 0.0001 +23.247785636564469 -0.04311132391388374 0.0001 +24.504422698000386 -0.040812383784974739 0.0001 +25.7610597594363 -0.038789286218401768 0.0001 +27.017696820872221 -0.037114935461959209 0.0001 +28.274333882308138 -0.035393491249622627 0.0001 +29.530970943744059 -0.033879643568396858 0.0001 diff --git a/test/regression/inputs/t_grid_half_lorentzian/case.param b/test/regression/inputs/t_grid_half_lorentzian/case.param index 110c996..2da8dbc 100644 --- a/test/regression/inputs/t_grid_half_lorentzian/case.param +++ b/test/regression/inputs/t_grid_half_lorentzian/case.param @@ -6,4 +6,4 @@ DATASPACE=frequency KERNEL=fermionic PARTICLE_HOLE_SYMMETRY=true DATA="data.dat" -FREQUENCY_GRID="half lorentzian" +FREQUENCY_GRID=half-lorentzian diff --git a/test/regression/make_inputs.py b/test/regression/make_inputs.py index cb8033a..eba4efa 100644 --- a/test/regression/make_inputs.py +++ b/test/regression/make_inputs.py @@ -306,9 +306,20 @@ def make(name, case_dir): else: base_frequency_ph(case_dir, name, options) elif name.startswith("t_grid_"): - grid = {"lorentzian": "lorentzian", "half_lorentzian": '"half lorentzian"', + grid = {"lorentzian": "lorentzian", "half_lorentzian": "half-lorentzian", "quadratic": "quadratic", "log": "log", "linear": "linear"}[name[len("t_grid_"):]] base_frequency_ph(case_dir, name, [("FREQUENCY_GRID", grid)]) + elif name in ("t_generate_err", "t_generate_err_seed"): + # Keep the scientific input identical so the two cases isolate SEED. + wn, y = freq_ph_data("t_generate_err") + write_columns(case_dir / "data.dat", wn, y, np.full_like(y, NOISE)) + entries = [("BETA", BETA), ("NFREQ", 40), ("N_ALPHA", 8), + ("NDAT", wn.size), ("DATASPACE", "frequency"), + ("KERNEL", "fermionic"), ("PARTICLE_HOLE_SYMMETRY", "true"), + ("DATA", '"data.dat"'), ("GENERATE_ERR", "true")] + if name == "t_generate_err_seed": + entries.append(("SEED", 1234)) + write_param(case_dir, entries) elif name in ("kk_imag_to_real_green", "kk_imag_to_real_self"): source, dataset, scale = { "kk_imag_to_real_green": ("u0_frequency", "files/in.out.avspec.dat", -np.pi), diff --git a/test/regression/manifest.py b/test/regression/manifest.py index d5ba99d..88bd198 100644 --- a/test/regression/manifest.py +++ b/test/regression/manifest.py @@ -87,7 +87,7 @@ def main(): "quadrature and, since step 2.2, the closed form used in maxent_kernel.cpp reproduce the GSL " "Legendre kernels of the references to 2e-16 and 3.3e-16.)", "* `-O0` builds take more than 30 minutes for the full `legendre` case.", - "* `*.spex.dat`, `*.fits.dat` and `*.booterr.dat` are not compared (see README.md).", ""] + "* `*.spex.dat` and `*.fits.dat` are not compared (see README.md).", ""] (HERE / "MANIFEST.md").write_text("\n".join(lines)) print(f"wrote MANIFEST.md: {len(cases)} cases, {total} datasets") diff --git a/test/regression/reference/cli_help.h5 b/test/regression/reference/cli_help.h5 index 9b86415..da90fd5 100644 Binary files a/test/regression/reference/cli_help.h5 and b/test/regression/reference/cli_help.h5 differ diff --git a/test/regression/reference/cli_help_grids.h5 b/test/regression/reference/cli_help_grids.h5 index a820e21..ebf022f 100644 Binary files a/test/regression/reference/cli_help_grids.h5 and b/test/regression/reference/cli_help_grids.h5 differ diff --git a/test/regression/reference/cli_missing_beta.h5 b/test/regression/reference/cli_missing_beta.h5 index 0c603d9..9051b9c 100644 Binary files a/test/regression/reference/cli_missing_beta.h5 and b/test/regression/reference/cli_missing_beta.h5 differ diff --git a/test/regression/reference/components.h5 b/test/regression/reference/components.h5 index 7717fff..088f09a 100644 Binary files a/test/regression/reference/components.h5 and b/test/regression/reference/components.h5 differ diff --git a/test/regression/reference/t_generate_err.h5 b/test/regression/reference/t_generate_err.h5 new file mode 100644 index 0000000..3f5710a Binary files /dev/null and b/test/regression/reference/t_generate_err.h5 differ diff --git a/test/regression/reference/t_generate_err_seed.h5 b/test/regression/reference/t_generate_err_seed.h5 new file mode 100644 index 0000000..7dca1ee Binary files /dev/null and b/test/regression/reference/t_generate_err_seed.h5 differ diff --git a/test/regression/reference/t_kernel_time_bosonic.h5 b/test/regression/reference/t_kernel_time_bosonic.h5 index b4d4029..316c6ea 100644 Binary files a/test/regression/reference/t_kernel_time_bosonic.h5 and b/test/regression/reference/t_kernel_time_bosonic.h5 differ diff --git a/test/regression/test_generate.py b/test/regression/test_generate.py new file mode 100644 index 0000000..25f62d7 --- /dev/null +++ b/test/regression/test_generate.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT + +import unittest + +from generate import classify_status + + +class ClassifyStatusTest(unittest.TestCase): + def test_timeout(self): + self.assertEqual(classify_status("maxent", None, ""), "timeout") + + def test_expected_maxent_exception(self): + self.assertEqual(classify_status("maxent", 1, "Caught Exception: bad input\n"), + "exception") + + def test_maxent_exit_without_signature_is_failure(self): + self.assertEqual(classify_status("maxent", 1, ""), "failed") + + def test_maxent_wrong_exit_with_signature_is_failure(self): + self.assertEqual(classify_status("maxent", 2, "Caught Exception: bad input\n"), + "failed") + + def test_success_ignores_diagnostic_text(self): + self.assertEqual(classify_status("maxent", 0, "Caught Exception: stale log\n"), "ok") + + def test_utility_nonzero_exit_is_failure(self): + self.assertEqual(classify_status("kk", 1, "Caught Exception: bad input\n"), "failed") + + +if __name__ == "__main__": + unittest.main()