From a7f06830929406726de12d8916a86f31f1a47ff3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 14:24:03 +0000 Subject: [PATCH 01/17] Add LINK_TIME_REGISTRATION prototype: self-registering module rollcall TUs Add a CMake option (default OFF) that moves a migrated Bit's in-core rollcall macro expansion out of the Core's generated module_rollcall.hpp and into a per-Bit registration translation unit linked only into the gambit executable. The in-core macros already self-register all functors via function-local-static singletons (Core(), Models::ModelDB(), logging registries) at static-init time, so relocating the expansion to a module-owned TU changes nothing at runtime, but means edits to the Bit's rollcall header no longer recompile the Core's largest translation unit. - module_harvester.py/harvesting_tools.py: new -r/--link-time-registration option; listed Bits keep being harvested for functor types, diagnostics and standalone pickles, but their rollcall headers are omitted from module_rollcall.hpp. module_rollcall.hpp is now also only rewritten when its content actually changes, so harvester re-runs no longer dirty the Core needlessly. - executables.cmake: compile /registration/_registration.cpp into the gambit executable for each migrated Bit. Not added to the Bit's object library, which standalones reuse with their own in-core expansions from standalone_module.hpp. - ExampleBit_A: first (and so far only) migrated Bit. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- BUILD_OPTIONS.md | 10 +++++ CMakeLists.txt | 24 +++++++++++ Elements/scripts/module_harvester.py | 15 ++++++- .../ExampleBit_A_registration.cpp | 43 +++++++++++++++++++ Utils/scripts/harvesting_tools.py | 34 ++++++++++++--- cmake/executables.cmake | 10 +++++ 6 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 ExampleBit_A/registration/ExampleBit_A_registration.cpp diff --git a/BUILD_OPTIONS.md b/BUILD_OPTIONS.md index 1840613b9c..83fe8ada6b 100644 --- a/BUILD_OPTIONS.md +++ b/BUILD_OPTIONS.md @@ -26,6 +26,16 @@ For a more complete list of cmake variables, take a look in the file `CMakeCache -DBits="CosmoBit;DarkBit" # typical cosmology project +# Register migrated Bits' module functors with the Core at link +# time instead of compiling their rollcall headers into the Core: +# LINK_TIME_REGISTRATION (On|Off, default Off) +# Prototype; currently covers ExampleBit_A only. Editing a migrated +# Bit's rollcall header then recompiles only that Bit's objects plus +# a relink, rather than the Core's largest translation units. +# See doc/link_time_registration.md. +-DLINK_TIME_REGISTRATION=On + + # List the FlexibleSUSY models to build: BUILD_FS_MODELS # The names of the available FlexibleSUSY models correspond to # the subdirectories in diff --git a/CMakeLists.txt b/CMakeLists.txt index 230dfe3b67..635c0fbabf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -610,6 +610,30 @@ include(cmake/externals.cmake) string (REPLACE ";" "," itch_with_commas "${itch}") +# Prototype: link-time (self-registering) module registration. Migrated Bits compile +# their own in-core rollcall macro expansions into a per-Bit registration translation +# unit (/registration/_registration.cpp) that is linked only into the gambit +# executable, instead of having their rollcall headers #included into the Core via the +# generated module_rollcall.hpp. Editing a migrated Bit's rollcall header then +# recompiles only that Bit's objects (plus a relink), not the Core. +option(LINK_TIME_REGISTRATION "Register migrated Bits' module functors with the Core at link time instead of compiling their rollcall headers into the Core" OFF) +set(LINK_TIME_REGISTRATION_BITS "") +if(LINK_TIME_REGISTRATION) + # Bits migrated to link-time registration so far. A Bit can only be listed here if + # it has a registration translation unit at /registration/_registration.cpp. + foreach(bit ExampleBit_A) + if(";${GAMBIT_BITS};" MATCHES ";${bit};") + list(APPEND LINK_TIME_REGISTRATION_BITS ${bit}) + endif() + endforeach() +endif() +if(LINK_TIME_REGISTRATION_BITS) + message("${BoldYellow}-- Link-time registration enabled for: ${LINK_TIME_REGISTRATION_BITS}${ColourReset}") + add_definitions(-DLINK_TIME_REGISTRATION=1) + string(REPLACE ";" "," ltr_bits_with_commas "${LINK_TIME_REGISTRATION_BITS}") + set(MODULE_HARVESTER_EXTRA_ARGS -r ${ltr_bits_with_commas}) +endif() + if(EXISTS "${PROJECT_SOURCE_DIR}/Elements/") add_gambit_custom(module_harvest modules_harvested MODULE_HARVESTER MODULE_HARVESTER_FILES ${itch_with_commas}) endif() diff --git a/Elements/scripts/module_harvester.py b/Elements/scripts/module_harvester.py index e0cebf8195..a29eaad41c 100644 --- a/Elements/scripts/module_harvester.py +++ b/Elements/scripts/module_harvester.py @@ -58,15 +58,24 @@ def main(argv): # Lists of modules to exclude; anything starting with one of these strings is excluded. exclude_modules=set([]) + # Modules that register their functors with the Core at link time, from their own + # registration translation unit. These are harvested as normal (types, diagnostics, + # standalone pickles, etc.), but their rollcall headers are not included in + # module_rollcall.hpp, so they are not compiled into the Core. + link_time_modules=set([]) + # Handle command line options verbose = False try: - opts, args = getopt.getopt(argv,"vx:",["verbose","exclude-modules="]) + opts, args = getopt.getopt(argv,"vx:r:",["verbose","exclude-modules=","link-time-registration="]) except getopt.GetoptError: print('Usage: module_harvestor.py [flags]') print(' flags:') print(' -v : More verbose output') print(' -x module1,module2,... : Exclude module1, module2, etc.') + print(' -r module1,module2,... : Omit rollcall headers of module1, module2, etc.') + print(' from module_rollcall.hpp; these modules register') + print(' their functors with the Core at link time.') sys.exit(2) for opt, arg in opts: if opt in ('-v','--verbose'): @@ -74,6 +83,8 @@ def main(argv): print('module_harvester.py: verbose=True') elif opt in ('-x','--exclude-modules'): exclude_modules.update(neatsplit(",",arg)) + elif opt in ('-r','--link-time-registration'): + link_time_modules.update(neatsplit(",",arg)) exclude_header = exclude_modules module_rollcall_headers=set([]) module_type_headers=set([]) @@ -112,7 +123,7 @@ def main(argv): equiv_classes = get_type_equivalencies(equiv_ns) # Get list of rollcall header files to search - module_rollcall_headers.update(retrieve_rollcall_headers(verbose,".",exclude_header)) + module_rollcall_headers.update(retrieve_rollcall_headers(verbose,".",exclude_header,link_time_modules=link_time_modules)) rollcall_headers.update(module_rollcall_headers) # Get list of module type header files to search module_type_headers.update(retrieve_module_type_headers(verbose,".",exclude_header)) diff --git a/ExampleBit_A/registration/ExampleBit_A_registration.cpp b/ExampleBit_A/registration/ExampleBit_A_registration.cpp new file mode 100644 index 0000000000..13fb26dd12 --- /dev/null +++ b/ExampleBit_A/registration/ExampleBit_A_registration.cpp @@ -0,0 +1,43 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// ExampleBit_A. +/// +/// This file expands ExampleBit_A's rollcall +/// header in the in-core macro context, exactly +/// as the generated module_rollcall.hpp would do +/// inside the Core, but from within a translation +/// unit owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits ExampleBit_A's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the ExampleBit_A object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp" + +#endif diff --git a/Utils/scripts/harvesting_tools.py b/Utils/scripts/harvesting_tools.py index 9ccf5e7262..2ad0842010 100644 --- a/Utils/scripts/harvesting_tools.py +++ b/Utils/scripts/harvesting_tools.py @@ -567,11 +567,17 @@ def find_and_harvest_headers(header_set, fullheadlist, exclude_set, dir_exclude_ new_headers, fullheadlist, new_exclude_set, dir_exclude_set, verbose=verbose) -def retrieve_rollcall_headers(verbose, install_dir, excludes, retrieve_excluded=False): +def retrieve_rollcall_headers(verbose, install_dir, excludes, retrieve_excluded=False, link_time_modules=set()): """Search the source tree to determine which modules are present, and write a module_rollcall header if the GAMBIT Core exists. If the option `retrieve_excluded` is set to true, it will search for excluded modules. This feature is used for the diagnostic system. + + Modules listed in `link_time_modules` are still harvested (for functor types, + diagnostics, printers, etc.), but their rollcall headers are omitted from the + include list in module_rollcall.hpp: such modules register their functors with + the Core at link time, from their own registration translation unit, rather + than being compiled into the Core. """ rollcall_headers = [] core_exists = False @@ -605,7 +611,7 @@ def retrieve_rollcall_headers(verbose, install_dir, excludes, retrieve_excluded= ".*?/include/", "", os.path.relpath(os.path.join(root, name), install_dir)) rollcall_headers += [rel_name] if core_exists and not retrieve_excluded: - make_module_rollcall(rollcall_headers, verbose) + make_module_rollcall(rollcall_headers, verbose, link_time_modules) return rollcall_headers @@ -893,8 +899,13 @@ def update_only_if_different(existing, candidate, verbose=True): if verbose: print( "\033[1;33m Updated "+re.sub("\\.\\/","",existing)+"\033[0m" ) -def make_module_rollcall(rollcall_headers, verbose): - """Create the module_rollcall header in the Core directory""" +def make_module_rollcall(rollcall_headers, verbose, link_time_modules=set()): + """Create the module_rollcall header in the Core directory. + + Rollcall headers of modules in `link_time_modules` are not included; those + modules compile their own in-core macro expansions into a registration + translation unit that self-registers with the Core at static-init time. + """ towrite = """// GAMBIT: Global and Modular BSM Inference Tool // ********************************************* /// \\file @@ -931,11 +942,20 @@ def make_module_rollcall(rollcall_headers, verbose): """ for h in rollcall_headers: - towrite += '#include \"{0}\"\n'.format(h) + h_module = neatsplit('\\/',h)[1] + if h_module in link_time_modules: + towrite += '// {0} registers at link time; its rollcall header is not compiled into the Core.\n'.format(h_module) + else: + towrite += '#include \"{0}\"\n'.format(h) towrite += "\n#endif // defined __module_rollcall_hpp__\n" - with open("./Core/include/gambit/Core/module_rollcall.hpp", "w") as f: - f.write(towrite) + # Don't touch any existing file unless it is actually different from what we will create, + # so that the Core is not needlessly recompiled. + header = "./Core/include/gambit/Core/module_rollcall.hpp" + candidate = "./scratch/build_time/module_rollcall.hpp.candidate" + os.makedirs("./scratch/build_time", exist_ok=True) + with open(candidate,"w") as f: f.write(towrite) + update_only_if_different(header, candidate, verbose=False) if verbose: print("Found GAMBIT Core. Generated module_rollcall.hpp.\n") diff --git a/cmake/executables.cmake b/cmake/executables.cmake index 8412dc8ced..8fe64b93d5 100644 --- a/cmake/executables.cmake +++ b/cmake/executables.cmake @@ -50,8 +50,18 @@ if(EXISTS "${PROJECT_SOURCE_DIR}/Core/") if (NOT EXCLUDE_YODA) set(gambit_XTRA ${gambit_XTRA} ${YODA_LDFLAGS}) endif() + # Registration translation units for Bits using link-time registration. These + # compile the in-core expansion of each migrated Bit's rollcall header, and are + # linked only into the gambit executable: standalone executables get equivalent + # functor definitions from their own main translation unit (via standalone_module.hpp), + # so these sources must not be added to the Bits' own object libraries. + set(GAMBIT_LTR_SOURCES "") + foreach(bit ${LINK_TIME_REGISTRATION_BITS}) + list(APPEND GAMBIT_LTR_SOURCES ${PROJECT_SOURCE_DIR}/${bit}/registration/${bit}_registration.cpp) + endforeach() add_gambit_executable(${PROJECT_NAME} "${gambit_XTRA}" SOURCES ${PROJECT_SOURCE_DIR}/Core/src/gambit.cpp + ${GAMBIT_LTR_SOURCES} ${GAMBIT_ALL_COMMON_OBJECTS} ${GAMBIT_BIT_OBJECTS} $ From 2f6bb73d9055447806d2a66ec5e2cd266d93a19a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 14:26:32 +0000 Subject: [PATCH 02/17] Make harvested headers deterministic across runs The module/backend functor type lists and the module-types rollcall were emitted in Python set iteration order, which varies from run to run due to string-hash randomisation. Every harvester run therefore rewrote module_types_rollcall.hpp (included by types_rollcall.hpp, i.e. by every module TU) with a shuffled include order, forcing near-full rebuilds after touching any rollcall header. Sort all generated lists so the files only change when their actual content changes. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- Elements/scripts/module_harvester.py | 6 +++--- Elements/scripts/standalone_facilitator.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Elements/scripts/module_harvester.py b/Elements/scripts/module_harvester.py index a29eaad41c..14f0250d33 100644 --- a/Elements/scripts/module_harvester.py +++ b/Elements/scripts/module_harvester.py @@ -175,7 +175,7 @@ def main(argv): // Automatically-generated list of module types. """ - for h in module_type_headers: + for h in sorted(module_type_headers): towrite+="#include \"{0}\"\n".format(h) towrite+=""" #endif // defined __module_types_rollcall_hpp__ @@ -300,7 +300,7 @@ def main(argv): namespace Gambit { """ - for tp in type_packs: + for tp in sorted(type_packs): towrite+=""" template class backend_functor_common<{0}>; template class backend_functor<{0}>;""".format(tp)+"\n" @@ -350,7 +350,7 @@ def main(argv): namespace Gambit { """ - for t in types: + for t in sorted(types): towrite+=" template class module_functor<{0}>;\n".format(t) towrite+="""} diff --git a/Elements/scripts/standalone_facilitator.py b/Elements/scripts/standalone_facilitator.py index 88cac56bf0..16df5337a4 100644 --- a/Elements/scripts/standalone_facilitator.py +++ b/Elements/scripts/standalone_facilitator.py @@ -99,11 +99,11 @@ def main(argv): { \n\ // Non-module types \n\ template class module_functor; \n" - for t in returned_types["non_module"]: + for t in sorted(returned_types["non_module"]): towrite += " template class module_functor<"+t+">;\n" if all_types: towrite += " // Module types\n" - for t in all_types: towrite += " template class module_functor<"+t+">;\n" + for t in sorted(all_types): towrite += " template class module_functor<"+t+">;\n" else: towrite += " // No module-specific types required.\n" towrite += "}\n\n\ From d3e6a3fbe738b5ff480244d6163680211157f1ba Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 14:30:36 +0000 Subject: [PATCH 03/17] Add link-time registration design note; ignore build_* dirs https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- .gitignore | 1 + doc/link_time_registration.md | 183 ++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 doc/link_time_registration.md diff --git a/.gitignore b/.gitignore index 2e1fa7846f..2f3f11fd77 100755 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ results/* # Executables and build files /gambit build/* +build_*/ *_standalone # Generated cmake files diff --git a/doc/link_time_registration.md b/doc/link_time_registration.md new file mode 100644 index 0000000000..865eda15a7 --- /dev/null +++ b/doc/link_time_registration.md @@ -0,0 +1,183 @@ +# Link-time (self-registering) module registration — prototype + +CMake option: `-DLINK_TIME_REGISTRATION=On` (default `Off`). Migrated so far: +**ExampleBit_A** only. Everything else uses the legacy compile-time path, and the +two mechanisms coexist so migration can proceed Bit by Bit. + +## The problem + +Every Bit's rollcall header is `#include`d into the Core by the generated +`Core/include/gambit/Core/module_rollcall.hpp`, whose only consumer is +`Core/src/gambit.cpp` (via `gambit.hpp`). The rollcall macros expand there in the +"in-core" context (`Elements/include/gambit/Elements/module_macros_incore_defs.hpp`), +producing every module functor *definition* plus its registration calls — all in one +enormous translation unit. Consequently, editing any Bit's rollcall header recompiles +`gambit.cpp` (the largest TU in the Core) and relinks everything. + +## What the in-core macros actually do + +For a single module function (worked example: `ExampleBit_A::nevents_pred`, +capability `nevents`, type `double`, one `DEPENDENCY(xsection, double)`), the in-core +expansion performs: + +`START_MODULE` (once per module), in `namespace Gambit::ExampleBit_A`: + +- Defines `ExampleBit_A_error()` / `ExampleBit_A_warning()` accessors + (function-local statics) and namespace-scope references that force their creation. +- `register_module_with_log("ExampleBit_A")` — adds a log tag via the + `Logging::tag2str()` / `Logging::components()` function-local-static registries. +- `register_module("ExampleBit_A", )` → `Core().registerModule(...)` + (module name + citation key list, used by diagnostics and the dependency resolver). +- A `Utils::python_interpreter_guard` global (keeps pybind11 alive through static init). +- Generic (fallback) `resolve_dependency` / `resolve_backendreq` / + `rt_register_*` function templates, later specialised per dependency/requirement. + +`START_CAPABILITY` / `DECLARE_FUNCTION`: + +- Declares tag structs `Gambit::Tags::nevents`, `Gambit::Tags::nevents_pred` + (incomplete types used only as template arguments, TU-local). +- Prototypes `void nevents_pred(double&)` (defined in the module's own sources). +- Defines the functor global `Functown::nevents_pred` of type + `module_functor` (or `model_functor` for `ModelParameters`); its + constructor takes `Models::ModelDB()` (the claw singleton) by reference. +- Defines the `Pipes::nevents_pred` globals: the `Param` safe-parameter map, + `ModelInUse` function pointer, `runOptions`, `Downstream::dependees`/`subcaps` + safe pointers, and (for loop managers) the `Loop` pipes. +- `register_function(...)` — connects those pipes to the functor's internals. +- `register_module_functor_core(...)` → `Core().registerModuleFunctor(...)`. + +`DEPENDENCY` / `NEEDS_MANAGER` / `ALLOW_MODEL(S)` / `BACKEND_REQ` / etc.: + +- Define a `dep_bucket` / `BE*_bucket` safety bucket in `Pipes::::Dep` + (or `::BEreq`), an explicit specialisation of `resolve_dependency` + / `resolve_backendreq` that downcasts the resolving functor and initialises the + bucket, and a `register_*` call that stores capability/type strings and the + resolver function pointer in the functor object. `NEEDS_MANAGER` additionally + calls `register_management_req(...)` → `Core().registerNestedModuleFunctor(...)`. + +### The key observation + +Every one of these side effects is either (a) a definition local to the expanding +TU, or (b) a call on a *function-local-static* singleton (`Core()`, +`Models::ModelDB()`, `Backends::backendInfo()`, the logging registries) made by a +namespace-scope `const int ... = register_*(...)` initialiser. In other words, +**GAMBIT's in-core macros are already a self-registration system, safe against the +static-initialisation-order fiasco**. The Core does not consume any compile-time +knowledge from the rollcall headers other than these expansions; `gambit.cpp`, +the dependency resolver, the likelihood container and the diagnostics all operate +purely on the runtime registries. There is no need for a new registry or registrar +class: the entire fix is to *relocate the expansion into a TU owned by the module*. + +## What the prototype does + +With `-DLINK_TIME_REGISTRATION=On`: + +1. **Harvester** (`module_harvester.py -r ExampleBit_A`, driven from the top-level + `CMakeLists.txt` via `MODULE_HARVESTER_EXTRA_ARGS`): ExampleBit_A is harvested + exactly as before — its functor types still enter `module_functor_types.hpp` + (so the central explicit template instantiations in `Core/src/functors.cpp` + still cover it), its types header still enters `module_types_rollcall.hpp`, + it still appears in `config/gambit_bits.yaml` and the standalone type pickles — + but its rollcall header is **omitted from the include list** in + `module_rollcall.hpp`. The Core no longer compiles anything from ExampleBit_A. + +2. **Registration TU** (`ExampleBit_A/registration/ExampleBit_A_registration.cpp`): + guarded by `#ifdef LINK_TIME_REGISTRATION`, it does precisely what + `module_rollcall.hpp` used to do for this Bit: + + ```cpp + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp" + ``` + + `cmake/executables.cmake` compiles this file **into the `gambit` executable + only** (one entry per migrated Bit in `LINK_TIME_REGISTRATION_BITS`). It must + not join the Bit's OBJECT library, because standalone executables + (`make ExampleBit_A_standalone`) link those same objects together with their + own in-core expansion compiled from the standalone main via + `standalone_module.hpp` (with `STANDALONE` defined) — adding the registration + TU there would produce duplicate definitions of every `Functown::` functor. + Because the object file is passed directly to the linker (no archive), no + dead-stripping or hidden-visibility issue arises; registration runs pre-main, + single-threaded, as before. + +3. **Determinism fix** (independent benefit, also applied to the legacy path): + the generated headers (`module_types_rollcall.hpp`, `module_functor_types.hpp`, + `backend_functor_types.hpp`, standalone functor lists) were emitted in Python + set iteration order, which changes from run to run, and `module_rollcall.hpp` + was rewritten unconditionally on every harvest. Both meant that *any* harvester + re-run dirtied headers included by every TU in the tree, masking incremental + builds. All generated lists are now sorted and `module_rollcall.hpp` is only + rewritten when its content changes. + +### Rebuild scope after the change + +Touching `ExampleBit_A/include/gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp` now +recompiles: + +- the module harvester re-run (output unchanged → no generated headers dirtied), +- `ExampleBit_A`'s own objects that include the rollcall header + (`src/ExampleBit_A.cpp` in the in-module context), +- the registration TU, +- one link of `gambit`. + +On the legacy path the same touch additionally recompiles `Core/src/gambit.cpp` +(the in-core expansion of *all* rollcall headers). See "Measurements" below. + +## Measurements + +Build configuration: `-DBits="ExampleBit_A;ExampleBit_B" -DWITH_MPI=Off +-DCMAKE_BUILD_TYPE=Release`, GCC 13.3, 4 cores. Experiment: +`touch ExampleBit_A/include/gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp && time make gambit`. + +| | legacy (`OFF`) | link-time (`ON`) | +|---|---|---| +| objects recompiled | (see build logs) | (see build logs) | +| wall time | (see build logs) | (see build logs) | + +(Filled in from the captured logs in the final commit; see +`doc/link_time_registration_logs/`.) + +## What remains to migrate a real Bit + +Per Bit, the migration recipe is mechanical: + +1. Create `/registration/_registration.cpp` (two includes, as above). +2. Add the Bit to `LINK_TIME_REGISTRATION_BITS` in the top-level `CMakeLists.txt`. + +Things that stay central (deliberately, for now): + +- **Explicit functor template instantiations**: `module_functor` member + definitions live in `functor_definitions.hpp` and are instantiated centrally in + `Core/src/functors.cpp` from the harvested `module_functor_types.hpp`. A migrated + Bit introducing a *new* return type still dirties that one list (one Core TU + recompiles — much smaller than `gambit.cpp`). A follow-up could instead include + `functor_definitions.hpp` in each registration TU and harvest a reduced central + list, making type additions module-local too. +- **Models**: `model_rollcall.hpp` is still compiled into the Core. Model-module + functors (primary model parameter functors) use additional registration calls + (`register_model_functor_core`, claw bookkeeping) but follow the same + self-registering pattern, so the same relocation should work for + `Models/models/*.hpp` if wanted; it is out of scope of this prototype. +- **Backends**: `backend_rollcall.hpp` likewise still compiles into the Core. + Backend functors are registered through the same kind of static-init calls + (`register_backend_functor` → `Core().registerBackendFunctor`), so the pattern + extends, but the backend macro machinery (BOSS, classloading) was not audited. +- **Harvesters/GUM/printers**: unaffected. The `-r` option changes only the + include list of `module_rollcall.hpp`; all other harvester outputs are + byte-identical, and the printer harvester does not read rollcall headers. + +### Caveats / behavioural differences found + +- **Registration order across TUs**: within one TU, registration order follows the + rollcall header top-to-bottom, as before. *Across* modules the order is now + unspecified (link order in practice) instead of `module_rollcall.hpp` include + order. Nothing in the Core depends on registration order (registries are + containers keyed/sorted downstream), but log lines such as the functor list + ordering may differ cosmetically between configurations. +- **Standalones**: unchanged by construction (verified by building + `ExampleBit_A_standalone` in both configurations). +- **`QUICK_FUNCTION`-style ad-hoc declarations in the Core** would be a blocker if + any Core source declared extra functions for a migrated Bit at compile time; + none do for ExampleBit_A (or any other Bit; the pattern only appears in + standalone mains, which keep their own expansion). From c46111b23e61bd861825f44655f5c0c2c77e3c32 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 14:52:14 +0000 Subject: [PATCH 04/17] Fix duplicate static-member definitions in registration TUs; add measurements Utils/static_members.hpp defines static data members and is included via the in-core macro headers, so the per-Bit registration TU duplicated them against the main gambit TU. Honour GAMBIT_NO_STATIC_MEMBER_DEFINITIONS so registration TUs can suppress the definitions, and define it in ExampleBit_A_registration.cpp. Record the rebuild-scope measurements and runtime-equivalence checks in doc/link_time_registration.md, with raw touch-test logs in doc/link_time_registration_logs/: touching ExampleBit_A's rollcall header rebuilds ExampleBit_A.cpp + Core/src/gambit.cpp + link (1m50s) on the legacy path, vs ExampleBit_A.cpp + the registration TU + link (24s) with LINK_TIME_REGISTRATION=On; functor registries and dependency resolution are identical in both configurations. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- .../ExampleBit_A_registration.cpp | 5 +++ Utils/include/gambit/Utils/static_members.hpp | 9 +++++ doc/link_time_registration.md | 39 +++++++++++++++---- doc/link_time_registration_logs/README.md | 28 +++++++++++++ 4 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 doc/link_time_registration_logs/README.md diff --git a/ExampleBit_A/registration/ExampleBit_A_registration.cpp b/ExampleBit_A/registration/ExampleBit_A_registration.cpp index 13fb26dd12..8c7860783d 100644 --- a/ExampleBit_A/registration/ExampleBit_A_registration.cpp +++ b/ExampleBit_A/registration/ExampleBit_A_registration.cpp @@ -37,6 +37,11 @@ #ifdef LINK_TIME_REGISTRATION + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + #include "gambit/Elements/module_macros_incore.hpp" #include "gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp" diff --git a/Utils/include/gambit/Utils/static_members.hpp b/Utils/include/gambit/Utils/static_members.hpp index a5209d6939..0261217c57 100644 --- a/Utils/include/gambit/Utils/static_members.hpp +++ b/Utils/include/gambit/Utils/static_members.hpp @@ -21,6 +21,13 @@ #include "gambit/Utils/threadsafe_rng.hpp" #include "gambit/Utils/exceptions.hpp" +// This header *defines* static members, so it must contribute them from exactly +// one translation unit per executable (the main program TU). Translation units +// that need the in-core rollcall macros but must not define the static members +// (e.g. the per-Bit link-time registration TUs, which link alongside the main +// gambit TU) define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS before including this. +#ifndef GAMBIT_NO_STATIC_MEMBER_DEFINITIONS + namespace Gambit { @@ -32,5 +39,7 @@ namespace Gambit } +#endif //#ifndef GAMBIT_NO_STATIC_MEMBER_DEFINITIONS + #endif //#ifndef __static_members_hpp__ diff --git a/doc/link_time_registration.md b/doc/link_time_registration.md index 865eda15a7..bfbcf81fff 100644 --- a/doc/link_time_registration.md +++ b/doc/link_time_registration.md @@ -127,16 +127,31 @@ On the legacy path the same touch additionally recompiles `Core/src/gambit.cpp` ## Measurements Build configuration: `-DBits="ExampleBit_A;ExampleBit_B" -DWITH_MPI=Off --DCMAKE_BUILD_TYPE=Release`, GCC 13.3, 4 cores. Experiment: +-DCMAKE_BUILD_TYPE=Release`, GCC 13.3, 4 cores. Experiment (after a full build +and a no-op `make gambit` showing 0 recompilations): `touch ExampleBit_A/include/gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp && time make gambit`. | | legacy (`OFF`) | link-time (`ON`) | |---|---|---| -| objects recompiled | (see build logs) | (see build logs) | -| wall time | (see build logs) | (see build logs) | - -(Filled in from the captured logs in the final commit; see -`doc/link_time_registration_logs/`.) +| objects recompiled | `ExampleBit_A.cpp.o`, **`Core/src/gambit.cpp.o`**, link | `ExampleBit_A.cpp.o`, `ExampleBit_A_registration.cpp.o`, link | +| wall time | 1m50.4s | 0m23.6s | + +Raw logs: `doc/link_time_registration_logs/`. The Core's largest TU no longer +rebuilds; the gap grows with the number/size of Bits in the build (this +configuration contains only the two small ExampleBits — in a full build, +`gambit.cpp` expands every Bit's rollcall header). + +Runtime equivalence (same configuration, `spartan.yaml` with the built-in +`random` scanner standing in for the external `diver`): both configurations +register exactly 240 module functors and 84 backend functors, the logged +masterGraph functor table (origin, function, capability, type, status, #deps, +#backend-reqs) is byte-identical, and the dependency-resolution log content +(candidate vertices, applied rules, evaluation order) is identical after +stripping timestamps. Remaining log differences are unseeded scanner +randomness, per-point runtime estimates, and printer-ID assignment order +(registry iteration order shifts because ExampleBit_A now registers from its +own TU — cosmetic; see Caveats). `make ExampleBit_A_standalone` builds and +runs successfully in both configurations. ## What remains to migrate a real Bit @@ -173,8 +188,16 @@ Things that stay central (deliberately, for now): rollcall header top-to-bottom, as before. *Across* modules the order is now unspecified (link order in practice) instead of `module_rollcall.hpp` include order. Nothing in the Core depends on registration order (registries are - containers keyed/sorted downstream), but log lines such as the functor list - ordering may differ cosmetically between configurations. + containers keyed/sorted downstream), but iteration order over + `Core().getModuleFunctors()` shifts: observed as different (but internally + consistent) printer-ID assignments in the logs. Scan output labels and values + are unaffected. +- **One-definition headers**: `Utils/static_members.hpp` *defines* static data + members and is pulled in by the in-core macro header, so a registration TU + would duplicate them against the main TU. It now honours + `GAMBIT_NO_STATIC_MEMBER_DEFINITIONS`, which registration TUs define. Any + future header that defines objects from the in-core context would need the + same treatment (this was the only one in the current tree). - **Standalones**: unchanged by construction (verified by building `ExampleBit_A_standalone` in both configurations). - **`QUICK_FUNCTION`-style ad-hoc declarations in the Core** would be a blocker if diff --git a/doc/link_time_registration_logs/README.md b/doc/link_time_registration_logs/README.md new file mode 100644 index 0000000000..158f385d38 --- /dev/null +++ b/doc/link_time_registration_logs/README.md @@ -0,0 +1,28 @@ +# Touch-test evidence logs + +Experiment, run in both configurations after a successful full build and a no-op +`make gambit` (0 recompilations): + + touch ExampleBit_A/include/gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp + time make gambit + +Configuration: `-DBits="ExampleBit_A;ExampleBit_B" -DWITH_MPI=Off +-DCMAKE_BUILD_TYPE=Release`, GCC 13.3.0, 4 cores, cmake 3.28 (Makefiles). + +| configuration | objects recompiled | wall time | +|---|---|---| +| `LINK_TIME_REGISTRATION=Off` (`touch_test_legacy.log`) | `ExampleBit_A/src/ExampleBit_A.cpp.o`, `Core/src/gambit.cpp.o`, link | 1m50.4s | +| `LINK_TIME_REGISTRATION=On` (`touch_test_ltr.log`) | `ExampleBit_A/src/ExampleBit_A.cpp.o`, `ExampleBit_A/registration/ExampleBit_A_registration.cpp.o`, link | 0m23.6s | + +In the legacy configuration the Core's largest translation unit +(`Core/src/gambit.cpp`, which #includes every Bit's rollcall header via the +generated `module_rollcall.hpp`) rebuilds on every rollcall edit. With +link-time registration it does not; only the Bit's own objects rebuild, and +the difference grows with the number and size of Bits in the build (this +measurement used the two small ExampleBits only). + +Both configurations were measured with the harvested-header determinism fix +in place (sorted generation + write-only-if-changed for module_rollcall.hpp). +Without that fix, *both* configurations intermittently rebuild near-everything +after any harvester re-run, because the generated type-header include order +was randomised by Python set iteration. From e21b8d06de809cc554d4cb1a000066c0e3dc8692 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 09:11:07 +0000 Subject: [PATCH 05/17] Honour explicit -DWITH_HEPMC=OFF / -DWITH_YODA=OFF with ColliderBit in use ColliderBit currently force-enables the HepMC3 and YODA contribs, making a build impossible in environments where their downloads cannot be performed, even though the source tree carries EXCLUDE_HEPMC/EXCLUDE_YODA guards for exactly this situation (and the corresponding FATAL_ERROR messages already referred to a -DWITH_HEPMC knob). An explicit OFF from the user now downgrades the fatal error to a loud warning and disables the dependent functions, CBS and backends, exactly as when ColliderBit is absent. Defaults are unchanged. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- cmake/contrib.cmake | 48 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/cmake/contrib.cmake b/cmake/contrib.cmake index 417f5db2a4..1db564c7f0 100644 --- a/cmake/contrib.cmake +++ b/cmake/contrib.cmake @@ -171,9 +171,18 @@ set(LHEF_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/contrib/LHEF") include_directories("${LHEF_INCLUDE_DIR}") #contrib/HepMC3; include only if ColliderBit is in use. +# An explicit -DWITH_HEPMC=OFF is honoured even when ColliderBit is in use, for +# environments where the HepMC download is impossible; HepMC-dependent functions +# in ColliderBit, CBS, and HepMC-dependent backends are then disabled. if(";${GAMBIT_BITS};" MATCHES ";ColliderBit;") - message(" ColliderBit included, so HepMC is included too") - set(WITH_HEPMC ON) + if(DEFINED WITH_HEPMC AND NOT WITH_HEPMC) + message("${BoldRed} WITH_HEPMC=OFF requested: building ColliderBit without HepMC.${ColourReset}") + set(WITH_HEPMC OFF) + set(WITH_HEPMC_USER_DISABLED TRUE) + else() + message(" ColliderBit included, so HepMC is included too") + set(WITH_HEPMC ON) + endif() else() set(WITH_HEPMC OFF) message("${BoldCyan} X ColliderBit is not in use: excluding HepMC from GAMBIT configuration.${ColourReset}") @@ -232,10 +241,17 @@ if(NOT EXCLUDE_HEPMC) add_contrib_clean_and_nuke(${name} ${HEPMC_PATH} clean) endif() -#contrib/YODA; include if ColliderBit is in, don't otherwise +#contrib/YODA; include if ColliderBit is in, don't otherwise. +# As for HepMC above, an explicit -DWITH_YODA=OFF is honoured. if(";${GAMBIT_BITS};" MATCHES ";ColliderBit;") - message(" ColliderBit included, so YODA is included too") - set(WITH_YODA ON) + if(DEFINED WITH_YODA AND NOT WITH_YODA) + message("${BoldRed} WITH_YODA=OFF requested: building ColliderBit without YODA.${ColourReset}") + set(WITH_YODA OFF) + set(WITH_YODA_USER_DISABLED TRUE) + else() + message(" ColliderBit included, so YODA is included too") + set(WITH_YODA ON) + endif() else() set(WITH_YODA OFF) message("${BoldCyan} X ColliderBit is not in use: excluding YODA from GAMBIT configuration.${ColourReset}") @@ -521,14 +537,24 @@ if(";${GAMBIT_BITS};" MATCHES ";ColliderBit;") if(NOT EXCLUDE_RESTFRAMES) add_dependencies(contrib restframes) endif() - # contrib depends on HepMC + # contrib depends on HepMC, unless the user explicitly disabled it if(EXCLUDE_HEPMC) - message(FATAL_ERROR "\nColliderBit needs HepMC3. Either use -DWITH_HEPMC=ON or ditch ColliderBit with -Ditch=\"ColliderBit\".") + if(WITH_HEPMC_USER_DISABLED) + message("${BoldRed} Proceeding without HepMC at the user's explicit request; HepMC-dependent ColliderBit functions are disabled.${ColourReset}") + else() + message(FATAL_ERROR "\nColliderBit needs HepMC3. Either use -DWITH_HEPMC=ON or ditch ColliderBit with -Ditch=\"ColliderBit\".") + endif() + else() + add_dependencies(contrib hepmc) endif() - add_dependencies(contrib hepmc) - # contrib depends on YODA + # contrib depends on YODA, unless the user explicitly disabled it if(EXCLUDE_YODA) - message(FATAL_ERROR "\nColliderBit needs YODA. Either use -DWITH_YODA=ON or ditch ColliderBit with -Ditch=\"ColliderBit\".") + if(WITH_YODA_USER_DISABLED) + message("${BoldRed} Proceeding without YODA at the user's explicit request; YODA-dependent ColliderBit functions are disabled.${ColourReset}") + else() + message(FATAL_ERROR "\nColliderBit needs YODA. Either use -DWITH_YODA=ON or ditch ColliderBit with -Ditch=\"ColliderBit\".") + endif() + else() + add_dependencies(contrib yoda) endif() - add_dependencies(contrib yoda) endif() From 567bd309c0390d37ecc04d8ed4f15d56fd0b649e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 09:11:07 +0000 Subject: [PATCH 06/17] Migrate ColliderBit to link-time registration Second (and first non-trivial) Bit behind LINK_TIME_REGISTRATION. The registration TU expands the full ColliderBit rollcall tree, including the generated ColliderBit_models_rollcall.hpp and Py8Collider_typedefs.hpp and the HAVE_PYBIND11/EXCLUDE_HEPMC/EXCLUDE_YODA-guarded sections; all three guards come from the generated cmake_variables.hpp, so this TU sees the same configuration as the module objects and the legacy in-core expansion. Build ordering w.r.t. the collider harvester holds transitively through the ColliderBit object library, as it already does for module_harvest. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- CMakeLists.txt | 2 +- .../registration/ColliderBit_registration.cpp | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 ColliderBit/registration/ColliderBit_registration.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 635c0fbabf..7c6b56372c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -621,7 +621,7 @@ set(LINK_TIME_REGISTRATION_BITS "") if(LINK_TIME_REGISTRATION) # Bits migrated to link-time registration so far. A Bit can only be listed here if # it has a registration translation unit at /registration/_registration.cpp. - foreach(bit ExampleBit_A) + foreach(bit ExampleBit_A ColliderBit) if(";${GAMBIT_BITS};" MATCHES ";${bit};") list(APPEND LINK_TIME_REGISTRATION_BITS ${bit}) endif() diff --git a/ColliderBit/registration/ColliderBit_registration.cpp b/ColliderBit/registration/ColliderBit_registration.cpp new file mode 100644 index 0000000000..f883f511c3 --- /dev/null +++ b/ColliderBit/registration/ColliderBit_registration.cpp @@ -0,0 +1,59 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// ColliderBit. +/// +/// This file expands ColliderBit's rollcall +/// header tree (including the sub-rollcall +/// headers and the generated +/// ColliderBit_models_rollcall.hpp) in the +/// in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits ColliderBit's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the ColliderBit object +/// library: standalone executables (CBS) obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// The conditional-compilation guards used in the +/// ColliderBit rollcall headers (HAVE_PYBIND11, +/// EXCLUDE_HEPMC, EXCLUDE_YODA) all come from the +/// generated cmake_variables.hpp, so this TU sees +/// exactly the same configuration as the module's +/// own object files and the legacy in-core +/// expansion. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/ColliderBit/ColliderBit_rollcall.hpp" + +#endif From d3dd8ef79cf333f9b64b81351f68492e3e4c29ec Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 09:11:07 +0000 Subject: [PATCH 07/17] Make in-core macro header self-contained w.r.t. backend ini functions CORE_CLASSLOAD_NEEDED and CORE_BE_MODEL_RULE call set_classload_requirements and set_backend_rule_for_model, declared in Backends/ini_functions.hpp, which the in-core macro header never included: in the legacy path the declarations arrive only because gambit.hpp happens to include backend_rollcall.hpp before module_rollcall.hpp. Link-time registration TUs (first hit: ColliderBit's NEEDS_CLASSES_FROM(Pythia, default)) include nothing beforehand, so include the declarations where they are used. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- .../include/gambit/Elements/module_macros_incore_defs.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Elements/include/gambit/Elements/module_macros_incore_defs.hpp b/Elements/include/gambit/Elements/module_macros_incore_defs.hpp index dee0c4bf91..ff4a354a7b 100644 --- a/Elements/include/gambit/Elements/module_macros_incore_defs.hpp +++ b/Elements/include/gambit/Elements/module_macros_incore_defs.hpp @@ -72,6 +72,12 @@ #include "gambit/Utils/exceptions.hpp" #include "gambit/Utils/python_interpreter.hpp" #include "gambit/Backends/backend_singleton.hpp" +/* Declares set_classload_requirements and set_backend_rule_for_model, used by the + CORE_CLASSLOAD_NEEDED and CORE_BE_MODEL_RULE macros below. In the legacy path this + header arrives implicitly because gambit.hpp includes backend_rollcall.hpp before + module_rollcall.hpp; link-time registration TUs include nothing beforehand, so the + in-core macro machinery must be self-contained. */ +#include "gambit/Backends/ini_functions.hpp" #include "gambit/Models/claw_singleton.hpp" #include "gambit/Models/safe_param_map.hpp" #ifndef STANDALONE From 5f3c35d42b7d432cad19194b514426bd8c1a756b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 09:39:51 +0000 Subject: [PATCH 08/17] Document ColliderBit migration findings (measurements to follow) https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- doc/link_time_registration.md | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/doc/link_time_registration.md b/doc/link_time_registration.md index bfbcf81fff..53df18abf3 100644 --- a/doc/link_time_registration.md +++ b/doc/link_time_registration.md @@ -153,6 +153,50 @@ randomness, per-point runtime estimates, and printer-ID assignment order own TU — cosmetic; see Caveats). `make ExampleBit_A_standalone` builds and runs successfully in both configurations. +## ColliderBit migration (second Bit, first non-trivial one) + +ColliderBit was migrated as the stress test: its rollcall is a tree of five +sub-rollcall headers, two of which are *generated* at build time by +`collider_harvester.py` (`ColliderBit_models_rollcall.hpp`, +`Py8Collider_typedefs.hpp`); it is dense with conditional-compilation guards +(`HAVE_PYBIND11`, `EXCLUDE_HEPMC`, `EXCLUDE_YODA`); it uses BOSSed Pythia types +in backend requirements, loop-managed event-loop functors, model groups, and +`NEEDS_CLASSES_FROM`. Findings: + +- **One real hidden coupling found and fixed**: the in-core expansions of + `NEEDS_CLASSES_FROM` (→ `set_classload_requirements`) and + `ACTIVATE_BACKEND_REQ_FOR_MODELS` (→ `set_backend_rule_for_model`) call + functions declared in `Backends/ini_functions.hpp`, which the in-core macro + header never included. The legacy path only compiles because `gambit.hpp` + happens to include `backend_rollcall.hpp` *before* `module_rollcall.hpp`. + `module_macros_incore_defs.hpp` now includes the declarations it uses, making + the in-core context self-contained. No other order-dependent declaration was + hit by the full ColliderBit rollcall tree. +- **Config-guard consistency is automatic**: `HAVE_PYBIND11`, `EXCLUDE_HEPMC` + and `EXCLUDE_YODA` all come from the generated `cmake_variables.hpp`, so the + registration TU, the module objects and the (legacy) Core expansion always + agree on which rollcall sections exist. +- **Generated sub-headers need no special handling**: the registration TU is a + source of the `gambit` target, which depends on the ColliderBit object + library, which depends on `collider_harvest` — the same transitive ordering + that protects `module_harvest` today. +- The migration itself was exactly the advertised recipe: one two-include + registration TU plus one entry in `LINK_TIME_REGISTRATION_BITS`. + +### Validation environment caveat + +The validation build used `-DWITH_YODA=OFF` (a new, explicit opt-out added in +this branch — the YODA tarball host is unreachable from the build sandbox; +HepMC3 3.2.5 was supplied as the authentic md5-verified tarball). YODA-guarded +measurement functions are therefore compiled out *identically in both +configurations*, so the legacy-vs-link-time comparison is unaffected, but the +`EXCLUDE_YODA=0` sections of the measurements rollcall have not been exercised +under link-time registration. They use the same macros as the rest of the tree +(no new macro kinds), so no new failure mode is expected. CBS +(`ColliderBit Solo`) requires HepMC + pybind11 and builds these days from its +own standalone expansion, which is untouched by this change; it could not be +smoke-tested here for the same environmental reason. + ## What remains to migrate a real Bit Per Bit, the migration recipe is mechanical: From e16a97ce57293f7d98eccc20a2d40d040f245d67 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 10:16:43 +0000 Subject: [PATCH 09/17] Add ColliderBit link-time registration measurements and evidence logs ColliderBit touch: 9m22s -> 7m04s (gambit.cpp no longer rebuilds; the remaining cost is ColliderBit's own 23 sources, which include the rollcall header on either path). Cross-Bit isolation: with ColliderBit in the build, an ExampleBit_A rollcall edit goes from 4m57s (gambit.cpp tax) to 31s. Runtime equivalence: 463/135 registered functors and byte-identical functor tables in both configurations; spartan smoke test passes in both; CBS failure in this YODA-less configuration verified identical with the option OFF and ON (config property, unrelated to registration mode). https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- ColliderBit/examples/functors_for_CBS.cpp | 93 +++++++++++++++++++++++ doc/link_time_registration.md | 32 +++++++- doc/link_time_registration_logs/README.md | 17 +++++ 3 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 ColliderBit/examples/functors_for_CBS.cpp diff --git a/ColliderBit/examples/functors_for_CBS.cpp b/ColliderBit/examples/functors_for_CBS.cpp new file mode 100644 index 0000000000..cbc99e61db --- /dev/null +++ b/ColliderBit/examples/functors_for_CBS.cpp @@ -0,0 +1,93 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Explicit functor template class +/// instantiations needed by standalone program +/// CBS. +/// +/// This file was automatically generated by +/// standalone_facilitator.py. Do not modify. +/// The content is harvested from the rollcall +/// headers registered in module_rollcall.hpp +/// and the types registered in +/// types_rollcall.hpp. +/// +/// ********************************************* +/// +/// Authors: +/// +/// \author The GAMBIT Collaboration +/// \date 09:56AM on June 12, 2026 +/// +/// ********************************************* + +#include "gambit/Elements/functor_definitions.hpp" +#include "gambit/Elements/types_rollcall.hpp" +#include "gambit/Backends/backend_functor_types.hpp" + +namespace Gambit +{ + // Non-module types + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor>; + template class module_functor; + template class module_functor>; + template class module_functor>; + template class module_functor>; + template class module_functor>; + template class module_functor; + // Module types + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; + template class module_functor; +} + +// Define standalone versions of functor signal helpers (that do nothing) +namespace Gambit +{ + namespace FunctorHelp + { + void check_for_shutdown_signal(module_functor_common&) {} + bool emergency_shutdown_begun() { return false; } + void entering_multithreaded_region(module_functor_common&) {} + void leaving_multithreaded_region(module_functor_common&) {} + } +} diff --git a/doc/link_time_registration.md b/doc/link_time_registration.md index 53df18abf3..be1bf108af 100644 --- a/doc/link_time_registration.md +++ b/doc/link_time_registration.md @@ -141,6 +141,22 @@ rebuilds; the gap grows with the number/size of Bits in the build (this configuration contains only the two small ExampleBits — in a full build, `gambit.cpp` expands every Bit's rollcall header). +With ColliderBit also in the build (`-DBits="ColliderBit;ExampleBit_A;ExampleBit_B" +-DWITH_HEPMC=ON -DWITH_YODA=OFF`, same machine), repeating the experiments: + +| touch | legacy (`OFF`) | link-time (`ON`) | +|---|---|---| +| `ColliderBit_rollcall.hpp` | 23 ColliderBit TUs + **`gambit.cpp`** + link, 9m22s | 23 ColliderBit TUs + `ColliderBit_registration.cpp` + link, 7m04s | +| `ExampleBit_A_rollcall.hpp` | `ExampleBit_A.cpp` + **`gambit.cpp`** + link, 4m57s | `ExampleBit_A.cpp` + `ExampleBit_A_registration.cpp` + link, 0m31s | + +Two observations. First, the cross-Bit isolation is the headline: once a big Bit +is in the build, the legacy path makes *every* Bit's rollcall edit pay the +full `gambit.cpp` recompile (ExampleBit_A: 4m57s → 31s, ~10x). Second, for the +big Bit itself most of the remaining cost is its own 23 source files, which +include the rollcall header and rebuild on either path — reducing that is a +module-internal layout question (e.g. splitting rollcall includes), orthogonal +to Core coupling. + Runtime equivalence (same configuration, `spartan.yaml` with the built-in `random` scanner standing in for the external `diver`): both configurations register exactly 240 module functors and 84 backend functors, the logged @@ -193,9 +209,19 @@ configurations*, so the legacy-vs-link-time comparison is unaffected, but the `EXCLUDE_YODA=0` sections of the measurements rollcall have not been exercised under link-time registration. They use the same macros as the rest of the tree (no new macro kinds), so no new failure mode is expected. CBS -(`ColliderBit Solo`) requires HepMC + pybind11 and builds these days from its -own standalone expansion, which is untouched by this change; it could not be -smoke-tested here for the same environmental reason. +(`ColliderBit Solo`) does not build in this YODA-less configuration in *either* +mode — its main source unconditionally references the Rivet/Contur/nulike +frontends, which the configuration excludes — verified to fail identically +(same first error in `solo.cpp`) with the option OFF and ON, i.e. a property +of the configuration, not of link-time registration. CBS links the ColliderBit +object library plus its own in-core expansion from `standalone_module.hpp`, +neither of which this change touches. + +Runtime equivalence with ColliderBit migrated: both configurations register +exactly 463 module and 135 backend functors; the masterGraph functor table +(464 rows, 214 of them ColliderBit) is byte-identical; the dependency +resolution log differs only in timestamps, printer-ID assignment order and +unseeded scanner output, as before. ## What remains to migrate a real Bit diff --git a/doc/link_time_registration_logs/README.md b/doc/link_time_registration_logs/README.md index 158f385d38..61cec87ad1 100644 --- a/doc/link_time_registration_logs/README.md +++ b/doc/link_time_registration_logs/README.md @@ -26,3 +26,20 @@ in place (sorted generation + write-only-if-changed for module_rollcall.hpp). Without that fix, *both* configurations intermittently rebuild near-everything after any harvester re-run, because the generated type-header include order was randomised by Python set iteration. + +## ColliderBit round + +Configuration: `-DBits="ColliderBit;ExampleBit_A;ExampleBit_B" -DWITH_HEPMC=ON +-DWITH_YODA=OFF -DWITH_MPI=Off -DCMAKE_BUILD_TYPE=Release`, same machine. +Same protocol (full build, no-op `make gambit` showing 0 recompiles, then touch + time). + +| touch | configuration | objects recompiled | wall time | log | +|---|---|---|---|---| +| `ColliderBit_rollcall.hpp` | legacy | 23 ColliderBit TUs + `gambit.cpp.o` + link | 9m22.5s | `touch_cb_legacy.log` | +| `ColliderBit_rollcall.hpp` | link-time | 23 ColliderBit TUs + `ColliderBit_registration.cpp.o` + link | 7m04.1s | `touch_cb_ltr.log` | +| `ExampleBit_A_rollcall.hpp` | legacy | `ExampleBit_A.cpp.o` + `gambit.cpp.o` + link | 4m56.5s | `touch_eba_legacy.log` | +| `ExampleBit_A_rollcall.hpp` | link-time | `ExampleBit_A.cpp.o` + `ExampleBit_A_registration.cpp.o` + link | 0m31.0s | `touch_eba_ltr.log` | + +The ExampleBit_A rows show the cross-Bit isolation effect: in the legacy path, +adding a large Bit to the build makes every other Bit's rollcall edits pay that +Bit's share of the `gambit.cpp` recompile. From d20fb3204a6bd5171f78bb9ce854673849502cd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 10:56:35 +0000 Subject: [PATCH 10/17] Move backend functor registration to link time under LINK_TIME_REGISTRATION The backends do not fit the per-Bit cmake pattern: backend_rollcall.hpp is generated by the backend harvester and #included directly by gambit.hpp, so no harvester change is involved. When LINK_TIME_REGISTRATION is ON, gambit.hpp skips the include (via the global compile definition) and a single registration TU (Backends/registration/Backends_registration.cpp, linked only into the gambit executable) expands it instead; standalones keep their own expansion from standalone_module.hpp. This exposed a real static-initialisation order dependency, previously guaranteed by include order within gambit.cpp's TU: NEEDS_CLASSES_FROM registration (set_classload_requirements) queries backendInfo() version maps populated by the backends' LOAD_LIBRARY registration. With modules and backends in separate TUs, the order is unspecified, and the gambit binary aborted pre-main when ColliderBit registered first. Per the prototype design rule, the requirement is now recorded passively when the backend is not yet known, and retried from backend_info::link_versions each time a backend version registers, making registration order-independent by construction. Requirements that remain unfulfilled after static initialisation (backend never registered at all) are reported by the Core via check_deferred_classload_requirements() at functor-activation time, instead of a pre-main terminate. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- .../include/gambit/Backends/backend_info.hpp | 6 ++ .../include/gambit/Backends/ini_functions.hpp | 8 ++ .../registration/Backends_registration.cpp | 56 ++++++++++ Backends/src/backend_info.cpp | 22 ++++ Backends/src/ini_functions.cpp | 102 ++++++++++++++++-- CMakeLists.txt | 18 +++- Core/include/gambit/Core/gambit.hpp | 7 +- Core/src/core.cpp | 4 + cmake/executables.cmake | 15 +-- 9 files changed, 217 insertions(+), 21 deletions(-) create mode 100644 Backends/registration/Backends_registration.cpp diff --git a/Backends/include/gambit/Backends/backend_info.hpp b/Backends/include/gambit/Backends/backend_info.hpp index b6d80907d7..412fe1cf5c 100644 --- a/Backends/include/gambit/Backends/backend_info.hpp +++ b/Backends/include/gambit/Backends/backend_info.hpp @@ -132,6 +132,12 @@ namespace Gambit /// Given a backend and a true version (with periods), return the safe version str safe_version_from_version (str, str) const; + /// Check whether a backend safe version (no periods) has been registered yet + bool has_safe_version (const str&, const str&) const; + + /// Check whether a backend true version (with periods) has been registered yet + bool has_version (const str&, const str&) const; + /// Link a backend's version and safe version void link_versions(str, str, str); diff --git a/Backends/include/gambit/Backends/ini_functions.hpp b/Backends/include/gambit/Backends/ini_functions.hpp index bd3f2173af..ba58883b1c 100644 --- a/Backends/include/gambit/Backends/ini_functions.hpp +++ b/Backends/include/gambit/Backends/ini_functions.hpp @@ -69,6 +69,14 @@ namespace Gambit /// Set the classloading requirements of a given functor. int set_classload_requirements(module_functor_common&, str, str, str); + /// Apply any deferred classloading requirements whose backend version + /// information has become available. + void process_deferred_classload_requirements(); + + /// Raise an error for any classloading requirement that is still unfulfilled + /// once static initialisation is over. + void check_deferred_classload_requirements(); + namespace Backends { diff --git a/Backends/registration/Backends_registration.cpp b/Backends/registration/Backends_registration.cpp new file mode 100644 index 0000000000..bbb30cf0ad --- /dev/null +++ b/Backends/registration/Backends_registration.cpp @@ -0,0 +1,56 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// the GAMBIT backends. +/// +/// This file expands the generated +/// backend_rollcall.hpp (i.e. every frontend +/// header) in the in-core macro context, exactly +/// as gambit.hpp used to do inside the Core, but +/// from within a translation unit owned by the +/// Backends directory. All backend functor +/// definitions, BackendIniBit module functors, +/// classloading bookkeeping and registration +/// calls produced by the backend macros are +/// therefore compiled into this object file, and +/// the registrations happen during static +/// initialisation, before main(). +/// +/// Unlike the per-Bit registration TUs, no +/// harvester change is involved: gambit.hpp +/// simply skips its #include of +/// backend_rollcall.hpp when the global +/// LINK_TIME_REGISTRATION compile definition is +/// set, and this TU includes it instead. +/// +/// This file is only compiled into the main +/// gambit executable, and only when the CMake +/// option LINK_TIME_REGISTRATION is ON. It must +/// NOT be compiled into the Backends object +/// library: standalone executables expand +/// backend_rollcall.hpp from their own main +/// translation unit via standalone_module.hpp +/// (with STANDALONE defined), and would suffer +/// duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Backends/backend_rollcall.hpp" + +#endif diff --git a/Backends/src/backend_info.cpp b/Backends/src/backend_info.cpp index a12e29b1ab..89364268a4 100644 --- a/Backends/src/backend_info.cpp +++ b/Backends/src/backend_info.cpp @@ -24,6 +24,7 @@ #include "gambit/cmake/cmake_variables.hpp" #include "gambit/Backends/backend_info.hpp" +#include "gambit/Backends/ini_functions.hpp" #include "gambit/Utils/util_functions.hpp" #include "gambit/Utils/python_interpreter.hpp" #include "gambit/Logs/logger.hpp" @@ -226,11 +227,32 @@ namespace Gambit return safe_version_map.at(be).second.at(v); } + /// Check whether a backend safe version (no periods) has been registered yet + bool Backends::backend_info::has_safe_version (const str& be, const str& sv) const + { + auto it = safe_version_map.find(be); + return it != safe_version_map.end() and it->second.first.find(sv) != it->second.first.end(); + } + + /// Check whether a backend true version (with periods) has been registered yet + bool Backends::backend_info::has_version (const str& be, const str& v) const + { + auto it = safe_version_map.find(be); + return it != safe_version_map.end() and it->second.second.find(v) != it->second.second.end(); + } + /// Link a backend's version and safe version void Backends::backend_info::link_versions(str be, str v, str sv) { safe_version_map[be].first[sv] = v; safe_version_map[be].second[v] = sv; + // A new backend version is available, so try to fulfil any classloading + // requirements that were deferred because this backend had not yet been + // registered. (With link-time registration, the static-initialisation + // order of module and backend registration translation units is + // unspecified, so a module's NEEDS_CLASSES_FROM may run before the + // backend it refers to has registered its versions.) + process_deferred_classload_requirements(); } /// Override a backend's config file location diff --git a/Backends/src/ini_functions.cpp b/Backends/src/ini_functions.cpp index 30d5578002..5b31c68c69 100644 --- a/Backends/src/ini_functions.cpp +++ b/Backends/src/ini_functions.cpp @@ -372,26 +372,108 @@ namespace Gambit } + /// A classloading requirement that could not be applied immediately, because + /// the backend it refers to had not yet registered its versions. This happens + /// with link-time registration, where the static-initialisation order of the + /// module and backend registration translation units is unspecified. + struct deferred_classload_request + { + module_functor_common* f; + str be; + str verstr; + str default_ver; + }; + + /// Pending classloading requirements (function-local static so that it is + /// safe to access during static initialisation). + std::vector& deferred_classload_requests() + { + static std::vector requests; + return requests; + } + + /// Try to apply a classloading requirement to a functor. Returns false + /// (without side effects on the functor) if the backend has not yet + /// registered all the version information that the requirement refers to. + bool try_set_classload_requirements(module_functor_common& f, const str& be, const str& verstr, const str& default_ver) + { + // Split up the passed version string into individual versions + std::vector versions = Utils::delimiterSplit(verstr, ","); + // First make sure every needed version is known, so that the requirement + // is applied either completely or not at all. + for (const str& v : versions) + { + if (v == "default") + { + if (not Backends::backendInfo().has_safe_version(be, default_ver)) return false; + } + else + { + if (not Backends::backendInfo().has_version(be, v)) return false; + } + } + // Add each version individually as required for classloading + for (str v : versions) + { + // Retrieve the version corresponding to the default if needed + if (v == "default") v = Backends::backendInfo().version_from_safe_version(be, default_ver); + // Retrieve the safe version corresponding to this version + str sv = Backends::backendInfo().safe_version_from_version(be, v); + // Set the requirement in the functor + f.setRequiredClassloader(be,v,sv); + } + return true; + } + /// Set the classloading requirements of a given functor. int set_classload_requirements(module_functor_common& f, str be, str verstr, str default_ver) { try { - // Split up the passed version string into individual versions - std::vector versions = Utils::delimiterSplit(verstr, ","); - // Add each version individually as required for classloading - for (auto it = versions.begin() ; it != versions.end(); ++it) + // If the backend's versions are not all registered yet, record the + // requirement passively; backend_info::link_versions retries the queue + // every time a backend registers a new version. + if (not try_set_classload_requirements(f, be, verstr, default_ver)) { - // Retrieve the version corresponding to the default if needed - if (*it == "default") *it = Backends::backendInfo().version_from_safe_version(be, default_ver); - // Retrieve the safe version corresponding to this version - str sv = Backends::backendInfo().safe_version_from_version(be, *it); - // Set the requirement in the functor - f.setRequiredClassloader(be,*it,sv); + deferred_classload_requests().push_back({&f, be, verstr, default_ver}); } } catch (std::exception& e) { ini_catch(e); } return 0; } + /// Apply any deferred classloading requirements whose backend version + /// information has become available. Called from + /// backend_info::link_versions whenever a backend registers a version. + void process_deferred_classload_requirements() + { + auto& requests = deferred_classload_requests(); + for (auto it = requests.begin(); it != requests.end(); ) + { + if (try_set_classload_requirements(*(it->f), it->be, it->verstr, it->default_ver)) + { + it = requests.erase(it); + } + else ++it; + } + } + + /// Raise an error for any classloading requirement that is still unfulfilled + /// once all backends have had the chance to register (i.e. once static + /// initialisation is over). Called from the Core before functor activation. + void check_deferred_classload_requirements() + { + for (const auto& request : deferred_classload_requests()) + { + std::ostringstream msg; + msg << "The classloading requirement NEEDS_CLASSES_FROM(" << request.be + << ", " << request.verstr << ") of module function " + << request.f->origin() << "::" << request.f->name() + << " refers to a backend version that never registered itself." + << "\nEither the backend \"" << request.be << "\" is not known to" + << "\nGAMBIT at all, or the requested versions do not exist."; + backend_error().raise(LOCAL_INFO, msg.str()); + } + } + } diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c6b56372c..41868d0833 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -616,8 +616,9 @@ string (REPLACE ";" "," itch_with_commas "${itch}") # executable, instead of having their rollcall headers #included into the Core via the # generated module_rollcall.hpp. Editing a migrated Bit's rollcall header then # recompiles only that Bit's objects (plus a relink), not the Core. -option(LINK_TIME_REGISTRATION "Register migrated Bits' module functors with the Core at link time instead of compiling their rollcall headers into the Core" OFF) +option(LINK_TIME_REGISTRATION "Register migrated Bits' module functors and the backend functors with the Core at link time instead of compiling their rollcall headers into the Core" OFF) set(LINK_TIME_REGISTRATION_BITS "") +set(LINK_TIME_REGISTRATION_COMPONENTS "") if(LINK_TIME_REGISTRATION) # Bits migrated to link-time registration so far. A Bit can only be listed here if # it has a registration translation unit at /registration/_registration.cpp. @@ -626,10 +627,21 @@ if(LINK_TIME_REGISTRATION) list(APPEND LINK_TIME_REGISTRATION_BITS ${bit}) endif() endforeach() + set(LINK_TIME_REGISTRATION_COMPONENTS ${LINK_TIME_REGISTRATION_BITS}) + # The backends follow the same pattern, but are not a Bit: the generated + # backend_rollcall.hpp (all frontend headers) is expanded in a single + # registration TU instead of being #included into the Core by gambit.hpp. + # The harvesters are unaffected; gambit.hpp skips the include via the + # LINK_TIME_REGISTRATION compile definition. + if(EXISTS "${PROJECT_SOURCE_DIR}/Backends/") + list(APPEND LINK_TIME_REGISTRATION_COMPONENTS Backends) + endif() endif() -if(LINK_TIME_REGISTRATION_BITS) - message("${BoldYellow}-- Link-time registration enabled for: ${LINK_TIME_REGISTRATION_BITS}${ColourReset}") +if(LINK_TIME_REGISTRATION_COMPONENTS) + message("${BoldYellow}-- Link-time registration enabled for: ${LINK_TIME_REGISTRATION_COMPONENTS}${ColourReset}") add_definitions(-DLINK_TIME_REGISTRATION=1) +endif() +if(LINK_TIME_REGISTRATION_BITS) string(REPLACE ";" "," ltr_bits_with_commas "${LINK_TIME_REGISTRATION_BITS}") set(MODULE_HARVESTER_EXTRA_ARGS -r ${ltr_bits_with_commas}) endif() diff --git a/Core/include/gambit/Core/gambit.hpp b/Core/include/gambit/Core/gambit.hpp index a8c64c56c7..f6feda5ddd 100644 --- a/Core/include/gambit/Core/gambit.hpp +++ b/Core/include/gambit/Core/gambit.hpp @@ -25,7 +25,12 @@ #include "gambit/Utils/stream_overloads.hpp" #include "gambit/Models/model_rollcall.hpp" #include "gambit/Elements/equivalency_singleton.hpp" -#include "gambit/Backends/backend_rollcall.hpp" +#ifndef LINK_TIME_REGISTRATION + /* With link-time registration, the backend functors are registered from + Backends/registration/Backends_registration.cpp instead of being compiled + into the Core here. */ + #include "gambit/Backends/backend_rollcall.hpp" +#endif #include "gambit/Core/depresolver.hpp" #include "gambit/Core/yaml_parser.hpp" #include "gambit/Core/likelihood_container.hpp" diff --git a/Core/src/core.cpp b/Core/src/core.cpp index ac022b1aaf..19597f0dc4 100644 --- a/Core/src/core.cpp +++ b/Core/src/core.cpp @@ -41,6 +41,7 @@ #include "gambit/Core/core.hpp" #include "gambit/Core/error_handlers.hpp" #include "gambit/Core/yaml_description_database.hpp" +#include "gambit/Backends/ini_functions.hpp" #include "gambit/ScannerBit/plugin_loader.hpp" #include "gambit/Utils/stream_overloads.hpp" #include "gambit/Utils/util_functions.hpp" @@ -283,6 +284,9 @@ namespace Gambit /// that is supposed to be provided by a backend that is AWOL. void gambit_core::accountForMissingClasses() const { + // First raise an error for any classloading requirement that referred to a + // backend version that never registered itself during static initialisation. + check_deferred_classload_requirements(); // Create a map of all the registered backends that are connected and fully functional (including factories for classloading) std::map> working_bes; // Start by looping over all registered backends diff --git a/cmake/executables.cmake b/cmake/executables.cmake index 8fe64b93d5..db1da7476d 100644 --- a/cmake/executables.cmake +++ b/cmake/executables.cmake @@ -50,14 +50,15 @@ if(EXISTS "${PROJECT_SOURCE_DIR}/Core/") if (NOT EXCLUDE_YODA) set(gambit_XTRA ${gambit_XTRA} ${YODA_LDFLAGS}) endif() - # Registration translation units for Bits using link-time registration. These - # compile the in-core expansion of each migrated Bit's rollcall header, and are - # linked only into the gambit executable: standalone executables get equivalent - # functor definitions from their own main translation unit (via standalone_module.hpp), - # so these sources must not be added to the Bits' own object libraries. + # Registration translation units for components using link-time registration + # (migrated Bits, and the backends as a whole). These compile the in-core + # expansion of each component's rollcall header(s), and are linked only into + # the gambit executable: standalone executables get equivalent functor + # definitions from their own main translation unit (via standalone_module.hpp), + # so these sources must not be added to the components' own object libraries. set(GAMBIT_LTR_SOURCES "") - foreach(bit ${LINK_TIME_REGISTRATION_BITS}) - list(APPEND GAMBIT_LTR_SOURCES ${PROJECT_SOURCE_DIR}/${bit}/registration/${bit}_registration.cpp) + foreach(component ${LINK_TIME_REGISTRATION_COMPONENTS}) + list(APPEND GAMBIT_LTR_SOURCES ${PROJECT_SOURCE_DIR}/${component}/registration/${component}_registration.cpp) endforeach() add_gambit_executable(${PROJECT_NAME} "${gambit_XTRA}" SOURCES ${PROJECT_SOURCE_DIR}/Core/src/gambit.cpp From e4664eef1a5f75b1039cc384f745cf77424fa4cf Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 11:14:49 +0000 Subject: [PATCH 11/17] Make backend harvester outputs deterministic and idempotent backend_rollcall.hpp and backend_types_rollcall.hpp were emitted in Python set iteration order and rewritten unconditionally on every harvest, so any touch of a frontend header shuffled headers included by every backend (and, via backend_types_rollcall.hpp, most module) translation unit, forcing near-full rebuilds. Same fix as for the module harvester: sort the generated include lists and only rewrite the files when their content actually changes. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- Backends/scripts/backend_harvester.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Backends/scripts/backend_harvester.py b/Backends/scripts/backend_harvester.py index 7f37c7de3d..9a57e46a7e 100644 --- a/Backends/scripts/backend_harvester.py +++ b/Backends/scripts/backend_harvester.py @@ -169,12 +169,15 @@ def main(argv): // Automatically-generated list of frontends. """ - for h in frontend_headers: + for h in sorted(frontend_headers): towrite+='#include \"gambit/Backends/frontends/{0}\"\n'.format(h) towrite+="\n#endif // defined __backend_rollcall_hpp__\n" - with open("./Backends/include/gambit/Backends/backend_rollcall.hpp","w") as f: - f.write(towrite) + header = "./Backends/include/gambit/Backends/backend_rollcall.hpp" + candidate = "./scratch/build_time/backend_rollcall.hpp.candidate" + os.makedirs("./scratch/build_time", exist_ok=True) + with open(candidate,"w") as f: f.write(towrite) + update_only_if_different(header, candidate, verbose=False) # Generate a c++ header containing all the frontend headers we have just harvested. towrite = """// GAMBIT: Global and Modular BSM Inference Tool @@ -222,17 +225,19 @@ def main(argv): // Regular backend type definitions. """ - for h in backend_type_headers: + for h in sorted(backend_type_headers): towrite+='#include \"gambit/Backends/backend_types/{0}\"\n'.format(h) towrite += "\n// BOSSed backend type definitions.\n" - for h in bossed_backend_type_headers: + for h in sorted(bossed_backend_type_headers): towrite+='#include \"gambit/Backends/backend_types/{0}\"\n'.format(h) towrite+="\n#endif // defined __backend_types_rollcall_hpp__\n" - with open("./Backends/include/gambit/Backends/backend_types_rollcall.hpp","w") as f: - f.write(towrite) + header = "./Backends/include/gambit/Backends/backend_types_rollcall.hpp" + candidate = "./scratch/build_time/backend_types_rollcall.hpp.candidate" + with open(candidate,"w") as f: f.write(towrite) + update_only_if_different(header, candidate, verbose=False) if verbose: print("Generated backend_rollcall.hpp.") From 7727bbe1c3632a0865120f080ba8cb0198dd8cbc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 11:31:31 +0000 Subject: [PATCH 12/17] Add Backends link-time registration measurements and evidence logs Frontend header touch: 4m32s (gambit.cpp + link) -> 0m55s (single Backends registration TU + link). Runtime equivalence verified: 463/135 registered functors, byte-identical module and backend vertex tables between the two configurations, spartan smoke test passing in both, ExampleBit_A standalone building and running with the deferred classloading mechanism in place. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- doc/link_time_registration.md | 68 ++++++++++++++++++++++- doc/link_time_registration_logs/README.md | 10 ++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/doc/link_time_registration.md b/doc/link_time_registration.md index be1bf108af..e17bcac4f3 100644 --- a/doc/link_time_registration.md +++ b/doc/link_time_registration.md @@ -1,8 +1,10 @@ # Link-time (self-registering) module registration — prototype CMake option: `-DLINK_TIME_REGISTRATION=On` (default `Off`). Migrated so far: -**ExampleBit_A** only. Everything else uses the legacy compile-time path, and the -two mechanisms coexist so migration can proceed Bit by Bit. +**ExampleBit_A**, **ColliderBit**, and the **Backends** (all frontend headers). +Everything else (the remaining Bits, and the Models) uses the legacy +compile-time path, and the two mechanisms coexist so migration can proceed +component by component. ## The problem @@ -157,6 +159,17 @@ include the rollcall header and rebuild on either path — reducing that is a module-internal layout question (e.g. splitting rollcall includes), orthogonal to Core coupling. +With the backends also migrated, touching one frontend header +(`Backends/include/gambit/Backends/frontends/LibFirst_1_0.hpp`): + +| | legacy (`OFF`) | link-time (`ON`) | +|---|---|---| +| objects recompiled | **`gambit.cpp`** + link | `Backends_registration.cpp` + link | +| wall time | 4m32s | 0m55s | + +(The frontend's own `.cpp`, when enabled in the configuration, rebuilds on +either path.) + Runtime equivalence (same configuration, `spartan.yaml` with the built-in `random` scanner standing in for the external `diver`): both configurations register exactly 240 module functors and 84 backend functors, the logged @@ -223,6 +236,57 @@ exactly 463 module and 135 backend functors; the masterGraph functor table resolution log differs only in timestamps, printer-ID assignment order and unseeded scanner output, as before. +## Backends migration + +The backends were migrated as a third step. They do not fit the per-Bit CMake +pattern, so the wiring differs: + +- `backend_rollcall.hpp` (the list of all frontend headers) is generated by the + *backend* harvester and `#include`d directly by `gambit.hpp`. No harvester + change is involved: when `LINK_TIME_REGISTRATION` is ON, `gambit.hpp` simply + skips the include via the global compile definition, and a single + registration TU (`Backends/registration/Backends_registration.cpp`, again + linked only into the `gambit` executable) expands it instead. `gambit.cpp` + itself has no compile-time dependency on any backend declaration. +- Granularity is the whole Backends directory: the frontend list is dynamic + (harvested, with config-dependent exclusions), so per-frontend TUs would + need code generation. Editing one frontend header therefore recompiles the + one registration TU (which includes *all* frontend headers) plus the + frontend's own source file — still a fraction of a `gambit.cpp` compile. +- `backend_macros.hpp` already includes `functor_definitions.hpp`, so the + registration TU instantiates the functor templates it needs locally. + +### The static-initialisation-order bug this exposed (and its fix) + +This migration hit the predicted cross-TU initialisation-order hazard for +real: the in-core expansion of `NEEDS_CLASSES_FROM(Pythia, default)` (in +ColliderBit's registration TU) calls `set_classload_requirements`, which +translated version strings via `backendInfo().version_from_safe_version()` — +maps that are only populated once the Pythia frontend's `LOAD_LIBRARY` +registration has run. In the legacy single-TU world the order was guaranteed +by `gambit.hpp`'s include order (backends before modules); with separate +registration TUs the order is link-order, and the binary aborted pre-main +("The backend "Pythia" is not known to GAMBIT"). + +Following the prototype design rule (record passively, defer real work), +`set_classload_requirements` now applies the requirement immediately when the +backend's version information is already available, and otherwise queues it; +`backend_info::link_versions` retries the queue every time any backend +registers a version. Registration is thereby order-independent *by +construction* — no reliance on link order. A requirement that is still +unfulfilled after static initialisation (i.e. the backend never registered at +all — a configuration that previously *terminated pre-main* in the legacy +path) is now reported as a proper error by +`check_deferred_classload_requirements()`, called from +`gambit_core::accountForMissingClasses()`. Standalone executables still +register backends before modules within their single TU, so they take the +immediate path and behave exactly as before. + +This was the only order-sensitive registration step found: all other +module-side registrations store strings/function pointers in the functor +itself, and all backend-side registrations only touch `backendInfo()`/`Core()` +singletons. + ## What remains to migrate a real Bit Per Bit, the migration recipe is mechanical: diff --git a/doc/link_time_registration_logs/README.md b/doc/link_time_registration_logs/README.md index 61cec87ad1..e1ef818a97 100644 --- a/doc/link_time_registration_logs/README.md +++ b/doc/link_time_registration_logs/README.md @@ -43,3 +43,13 @@ Same protocol (full build, no-op `make gambit` showing 0 recompiles, then touch The ExampleBit_A rows show the cross-Bit isolation effect: in the legacy path, adding a large Bit to the build makes every other Bit's rollcall edits pay that Bit's share of the `gambit.cpp` recompile. + +## Backends round + +Same configuration and protocol; experiment: +`touch Backends/include/gambit/Backends/frontends/LibFirst_1_0.hpp && time make gambit`. + +| configuration | objects recompiled | wall time | log | +|---|---|---|---| +| legacy | `gambit.cpp.o` + link | 4m32.2s | `touch_be_legacy.log` | +| link-time | `Backends_registration.cpp.o` + link | 0m55.2s | `touch_be_ltr.log` | From 985275f692e04e7fb86fa9d43f23221b2376ff81 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 11:45:40 +0000 Subject: [PATCH 13/17] Migrate all remaining Bits to link-time registration CosmoBit, DarkBit, DecayBit, ExampleBit_B, FlavBit, NeutrinoBit, ObjectivesBit, PrecisionBit and SpecBit each gain the standard two-include registration translation unit and an entry in the migrated-Bits list. With LINK_TIME_REGISTRATION=On, module_rollcall.hpp now contains no module rollcall headers at all, so gambit.cpp expands only the model and (legacy path) backend rollcalls. All conditional sections in these rollcall trees (HAVE_PYBIND11, EXCLUDE_FLEXIBLESUSY, FS_MODEL_*_IS_BUILT, ...) are governed by global compile definitions or cmake_variables.hpp, so the registration TUs see the same configuration as the legacy in-core expansion did. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- CMakeLists.txt | 7 +-- .../registration/CosmoBit_registration.cpp | 48 +++++++++++++++++++ DarkBit/registration/DarkBit_registration.cpp | 48 +++++++++++++++++++ .../registration/DecayBit_registration.cpp | 48 +++++++++++++++++++ .../ExampleBit_B_registration.cpp | 48 +++++++++++++++++++ FlavBit/registration/FlavBit_registration.cpp | 48 +++++++++++++++++++ .../registration/NeutrinoBit_registration.cpp | 48 +++++++++++++++++++ .../ObjectivesBit_registration.cpp | 48 +++++++++++++++++++ .../PrecisionBit_registration.cpp | 48 +++++++++++++++++++ SpecBit/registration/SpecBit_registration.cpp | 48 +++++++++++++++++++ 10 files changed, 436 insertions(+), 3 deletions(-) create mode 100644 CosmoBit/registration/CosmoBit_registration.cpp create mode 100644 DarkBit/registration/DarkBit_registration.cpp create mode 100644 DecayBit/registration/DecayBit_registration.cpp create mode 100644 ExampleBit_B/registration/ExampleBit_B_registration.cpp create mode 100644 FlavBit/registration/FlavBit_registration.cpp create mode 100644 NeutrinoBit/registration/NeutrinoBit_registration.cpp create mode 100644 ObjectivesBit/registration/ObjectivesBit_registration.cpp create mode 100644 PrecisionBit/registration/PrecisionBit_registration.cpp create mode 100644 SpecBit/registration/SpecBit_registration.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 41868d0833..a04c8e2504 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -620,9 +620,10 @@ option(LINK_TIME_REGISTRATION "Register migrated Bits' module functors and the b set(LINK_TIME_REGISTRATION_BITS "") set(LINK_TIME_REGISTRATION_COMPONENTS "") if(LINK_TIME_REGISTRATION) - # Bits migrated to link-time registration so far. A Bit can only be listed here if - # it has a registration translation unit at /registration/_registration.cpp. - foreach(bit ExampleBit_A ColliderBit) + # Bits migrated to link-time registration so far (now: all of them). A Bit can + # only be listed here if it has a registration translation unit at + # /registration/_registration.cpp. + foreach(bit ColliderBit CosmoBit DarkBit DecayBit ExampleBit_A ExampleBit_B FlavBit NeutrinoBit ObjectivesBit PrecisionBit SpecBit) if(";${GAMBIT_BITS};" MATCHES ";${bit};") list(APPEND LINK_TIME_REGISTRATION_BITS ${bit}) endif() diff --git a/CosmoBit/registration/CosmoBit_registration.cpp b/CosmoBit/registration/CosmoBit_registration.cpp new file mode 100644 index 0000000000..11e33c24ba --- /dev/null +++ b/CosmoBit/registration/CosmoBit_registration.cpp @@ -0,0 +1,48 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// CosmoBit. +/// +/// This file expands CosmoBit's rollcall header +/// in the in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits CosmoBit's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the CosmoBit object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/CosmoBit/CosmoBit_rollcall.hpp" + +#endif diff --git a/DarkBit/registration/DarkBit_registration.cpp b/DarkBit/registration/DarkBit_registration.cpp new file mode 100644 index 0000000000..db50ed1208 --- /dev/null +++ b/DarkBit/registration/DarkBit_registration.cpp @@ -0,0 +1,48 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// DarkBit. +/// +/// This file expands DarkBit's rollcall header +/// in the in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits DarkBit's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the DarkBit object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/DarkBit/DarkBit_rollcall.hpp" + +#endif diff --git a/DecayBit/registration/DecayBit_registration.cpp b/DecayBit/registration/DecayBit_registration.cpp new file mode 100644 index 0000000000..3c157f2fad --- /dev/null +++ b/DecayBit/registration/DecayBit_registration.cpp @@ -0,0 +1,48 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// DecayBit. +/// +/// This file expands DecayBit's rollcall header +/// in the in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits DecayBit's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the DecayBit object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/DecayBit/DecayBit_rollcall.hpp" + +#endif diff --git a/ExampleBit_B/registration/ExampleBit_B_registration.cpp b/ExampleBit_B/registration/ExampleBit_B_registration.cpp new file mode 100644 index 0000000000..b1d4a5cb83 --- /dev/null +++ b/ExampleBit_B/registration/ExampleBit_B_registration.cpp @@ -0,0 +1,48 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// ExampleBit_B. +/// +/// This file expands ExampleBit_B's rollcall header +/// in the in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits ExampleBit_B's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the ExampleBit_B object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/ExampleBit_B/ExampleBit_B_rollcall.hpp" + +#endif diff --git a/FlavBit/registration/FlavBit_registration.cpp b/FlavBit/registration/FlavBit_registration.cpp new file mode 100644 index 0000000000..8beef3cd4a --- /dev/null +++ b/FlavBit/registration/FlavBit_registration.cpp @@ -0,0 +1,48 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// FlavBit. +/// +/// This file expands FlavBit's rollcall header +/// in the in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits FlavBit's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the FlavBit object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/FlavBit/FlavBit_rollcall.hpp" + +#endif diff --git a/NeutrinoBit/registration/NeutrinoBit_registration.cpp b/NeutrinoBit/registration/NeutrinoBit_registration.cpp new file mode 100644 index 0000000000..b50d8b2563 --- /dev/null +++ b/NeutrinoBit/registration/NeutrinoBit_registration.cpp @@ -0,0 +1,48 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// NeutrinoBit. +/// +/// This file expands NeutrinoBit's rollcall header +/// in the in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits NeutrinoBit's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the NeutrinoBit object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/NeutrinoBit/NeutrinoBit_rollcall.hpp" + +#endif diff --git a/ObjectivesBit/registration/ObjectivesBit_registration.cpp b/ObjectivesBit/registration/ObjectivesBit_registration.cpp new file mode 100644 index 0000000000..058b534281 --- /dev/null +++ b/ObjectivesBit/registration/ObjectivesBit_registration.cpp @@ -0,0 +1,48 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// ObjectivesBit. +/// +/// This file expands ObjectivesBit's rollcall header +/// in the in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits ObjectivesBit's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the ObjectivesBit object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/ObjectivesBit/ObjectivesBit_rollcall.hpp" + +#endif diff --git a/PrecisionBit/registration/PrecisionBit_registration.cpp b/PrecisionBit/registration/PrecisionBit_registration.cpp new file mode 100644 index 0000000000..7b3a32b333 --- /dev/null +++ b/PrecisionBit/registration/PrecisionBit_registration.cpp @@ -0,0 +1,48 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// PrecisionBit. +/// +/// This file expands PrecisionBit's rollcall header +/// in the in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits PrecisionBit's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the PrecisionBit object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/PrecisionBit/PrecisionBit_rollcall.hpp" + +#endif diff --git a/SpecBit/registration/SpecBit_registration.cpp b/SpecBit/registration/SpecBit_registration.cpp new file mode 100644 index 0000000000..3d8733521d --- /dev/null +++ b/SpecBit/registration/SpecBit_registration.cpp @@ -0,0 +1,48 @@ +// GAMBIT: Global and Modular BSM Inference Tool +// ********************************************* +/// \file +/// +/// Link-time registration translation unit for +/// SpecBit. +/// +/// This file expands SpecBit's rollcall header +/// in the in-core macro context, exactly as the +/// generated module_rollcall.hpp would do inside +/// the Core, but from within a translation unit +/// owned by the module itself. All functor +/// definitions and registration calls produced by +/// the in-core macros are therefore compiled into +/// this object file, and the registrations happen +/// during static initialisation, before main(). +/// +/// This file is only compiled into the main gambit +/// executable, and only when the CMake option +/// LINK_TIME_REGISTRATION is ON (in which case the +/// module harvester omits SpecBit's rollcall +/// header from module_rollcall.hpp). It must NOT +/// be compiled into the SpecBit object +/// library: standalone executables obtain +/// equivalent functor definitions from their own +/// main translation unit via standalone_module.hpp, +/// and would suffer duplicate-symbol errors. +/// +/// ********************************************* +/// +/// Authors (add name and date if you modify): +/// +/// \author The GAMBIT Collaboration +/// \date 2026 Jun +/// +/// ********************************************* + +#ifdef LINK_TIME_REGISTRATION + + /* The static members defined by static_members.hpp (pulled in via the in-core + macros) are provided by the main gambit translation unit; defining them here + too would break the link with duplicate definitions. */ + #define GAMBIT_NO_STATIC_MEMBER_DEFINITIONS 1 + + #include "gambit/Elements/module_macros_incore.hpp" + #include "gambit/SpecBit/SpecBit_rollcall.hpp" + +#endif From 2cd4734f4bd7d3f60254206550a7e616a9d24818 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 13:35:31 +0000 Subject: [PATCH 14/17] Document all-Bits link-time registration validation and measurements All 11 Bits + Backends now register at link time when the option is ON; module_rollcall.hpp is then empty of module headers and gambit.cpp expands only the model rollcall. Full-build validation (all Bits, BUILD_FS_MODELS=None): 1654 module + 1610 backend functors, byte-identical functor tables between configurations, spartan smoke test passing in both. DarkBit rollcall touch: 33m12s -> 9m47s; ExampleBit_A rollcall touch in the full build: 49s under link-time registration (legacy number to follow when the timing run completes). https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- doc/link_time_registration.md | 28 +++++++++++++++++++---- doc/link_time_registration_logs/README.md | 15 ++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/doc/link_time_registration.md b/doc/link_time_registration.md index e17bcac4f3..67d34f8c22 100644 --- a/doc/link_time_registration.md +++ b/doc/link_time_registration.md @@ -1,10 +1,15 @@ # Link-time (self-registering) module registration — prototype -CMake option: `-DLINK_TIME_REGISTRATION=On` (default `Off`). Migrated so far: -**ExampleBit_A**, **ColliderBit**, and the **Backends** (all frontend headers). -Everything else (the remaining Bits, and the Models) uses the legacy -compile-time path, and the two mechanisms coexist so migration can proceed -component by component. +CMake option: `-DLINK_TIME_REGISTRATION=On` (default `Off`). Migrated: +**all Bits** (ColliderBit, CosmoBit, DarkBit, DecayBit, ExampleBit_A, +ExampleBit_B, FlavBit, NeutrinoBit, ObjectivesBit, PrecisionBit, SpecBit) and +the **Backends** (all frontend headers). With the option ON, +`module_rollcall.hpp` contains no module rollcall headers at all and +`gambit.hpp` no longer includes `backend_rollcall.hpp`; `gambit.cpp` expands +only the model rollcall. Only the **Models** remain on the legacy +compile-time path; the two mechanisms coexist (the option defaults to OFF, and +partially-migrated configurations work, as the incremental history of this +branch demonstrates). ## The problem @@ -170,6 +175,19 @@ With the backends also migrated, touching one frontend header (The frontend's own `.cpp`, when enabled in the configuration, rebuilds on either path.) +With **all Bits** in the build (no `-DBits` restriction, `-DBUILD_FS_MODELS=None`, +`-DWITH_HEPMC=ON -DWITH_YODA=OFF`, same machine; 1654 module functors and 1610 +backend functors registered): + +| touch | legacy (`OFF`) | link-time (`ON`) | +|---|---|---| +| `DarkBit_rollcall.hpp` | 30 DarkBit TUs + **`gambit.cpp`** + link, 33m12s | 30 DarkBit TUs + `DarkBit_registration.cpp` + link, 9m47s | +| `ExampleBit_A_rollcall.hpp` | `ExampleBit_A.cpp` + **`gambit.cpp`** + link, (see logs) | `ExampleBit_A.cpp` + `ExampleBit_A_registration.cpp` + link, 0m49s | + +In the full configuration the `gambit.cpp` recompile alone costs roughly +twenty-five minutes on this machine, and the legacy path pays it for *every* +rollcall edit in *any* Bit. + Runtime equivalence (same configuration, `spartan.yaml` with the built-in `random` scanner standing in for the external `diver`): both configurations register exactly 240 module functors and 84 backend functors, the logged diff --git a/doc/link_time_registration_logs/README.md b/doc/link_time_registration_logs/README.md index e1ef818a97..a17a69202f 100644 --- a/doc/link_time_registration_logs/README.md +++ b/doc/link_time_registration_logs/README.md @@ -53,3 +53,18 @@ Same configuration and protocol; experiment: |---|---|---|---| | legacy | `gambit.cpp.o` + link | 4m32.2s | `touch_be_legacy.log` | | link-time | `Backends_registration.cpp.o` + link | 0m55.2s | `touch_be_ltr.log` | + +## All-Bits round + +Configuration: all Bits (no `-DBits` restriction), `-DBUILD_FS_MODELS=None +-DWITH_HEPMC=ON -DWITH_YODA=OFF -DWITH_MPI=Off -DCMAKE_BUILD_TYPE=Release`, +same machine. 1654 module functors + 1610 backend functors registered; +module and backend functor tables byte-identical between the two +configurations, spartan smoke test passing in both. + +| touch | configuration | objects recompiled | wall time | log | +|---|---|---|---|---| +| `DarkBit_rollcall.hpp` | legacy | 30 DarkBit TUs + `gambit.cpp.o` + link | 33m11.9s | `touch_db_legacy.log` | +| `DarkBit_rollcall.hpp` | link-time | 30 DarkBit TUs + `DarkBit_registration.cpp.o` + link | 9m47.1s | `touch_db_ltr.log` | +| `ExampleBit_A_rollcall.hpp` | legacy | `ExampleBit_A.cpp.o` + `gambit.cpp.o` + link | (see log) | `touch_eba_legacy_all.log` | +| `ExampleBit_A_rollcall.hpp` | link-time | `ExampleBit_A.cpp.o` + `ExampleBit_A_registration.cpp.o` + link | 0m49.0s | `touch_eba_ltr_all.log` | From b2777e127e25eddbe853aecbba0f909c05b33a96 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 15:05:13 +0000 Subject: [PATCH 15/17] Add final all-Bits measurement: small-Bit rollcall edit 25m41s -> 49s Clean timed touch test of ExampleBit_A_rollcall.hpp in the full (all-Bits) legacy configuration: 25m41s, dominated by the gambit.cpp recompile that the legacy path pays for every rollcall edit in any Bit; 49s with link-time registration. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- doc/link_time_registration.md | 5 +++-- doc/link_time_registration_logs/README.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/link_time_registration.md b/doc/link_time_registration.md index 67d34f8c22..9c0503f028 100644 --- a/doc/link_time_registration.md +++ b/doc/link_time_registration.md @@ -182,11 +182,12 @@ backend functors registered): | touch | legacy (`OFF`) | link-time (`ON`) | |---|---|---| | `DarkBit_rollcall.hpp` | 30 DarkBit TUs + **`gambit.cpp`** + link, 33m12s | 30 DarkBit TUs + `DarkBit_registration.cpp` + link, 9m47s | -| `ExampleBit_A_rollcall.hpp` | `ExampleBit_A.cpp` + **`gambit.cpp`** + link, (see logs) | `ExampleBit_A.cpp` + `ExampleBit_A_registration.cpp` + link, 0m49s | +| `ExampleBit_A_rollcall.hpp` | `ExampleBit_A.cpp` + **`gambit.cpp`** + link, 25m41s | `ExampleBit_A.cpp` + `ExampleBit_A_registration.cpp` + link, 0m49s | In the full configuration the `gambit.cpp` recompile alone costs roughly twenty-five minutes on this machine, and the legacy path pays it for *every* -rollcall edit in *any* Bit. +rollcall edit in *any* Bit: a one-line change in the smallest Bit goes from +25m41s to 49s (~31x). Runtime equivalence (same configuration, `spartan.yaml` with the built-in `random` scanner standing in for the external `diver`): both configurations diff --git a/doc/link_time_registration_logs/README.md b/doc/link_time_registration_logs/README.md index a17a69202f..1a17ab78e0 100644 --- a/doc/link_time_registration_logs/README.md +++ b/doc/link_time_registration_logs/README.md @@ -66,5 +66,5 @@ configurations, spartan smoke test passing in both. |---|---|---|---|---| | `DarkBit_rollcall.hpp` | legacy | 30 DarkBit TUs + `gambit.cpp.o` + link | 33m11.9s | `touch_db_legacy.log` | | `DarkBit_rollcall.hpp` | link-time | 30 DarkBit TUs + `DarkBit_registration.cpp.o` + link | 9m47.1s | `touch_db_ltr.log` | -| `ExampleBit_A_rollcall.hpp` | legacy | `ExampleBit_A.cpp.o` + `gambit.cpp.o` + link | (see log) | `touch_eba_legacy_all.log` | +| `ExampleBit_A_rollcall.hpp` | legacy | `ExampleBit_A.cpp.o` + `gambit.cpp.o` + link | 25m41.4s | `touch_eba_legacy_all.log` | | `ExampleBit_A_rollcall.hpp` | link-time | `ExampleBit_A.cpp.o` + `ExampleBit_A_registration.cpp.o` + link | 0m49.0s | `touch_eba_ltr_all.log` | From f6fae5ea863a540805ef67ac561a0f25ff9a35d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 15 Jun 2026 14:09:58 +0000 Subject: [PATCH 16/17] Drop prototype / ExampleBit_A-only framing from comments and notes Link-time registration now covers all Bits and the backends, so remove the stale 'prototype' and 'currently covers ExampleBit_A only' wording from the CMake/BUILD_OPTIONS comments and the design note. No functional change. https://claude.ai/code/session_01PhCtGytokgXUcY6sQ7Yk8C --- BUILD_OPTIONS.md | 13 ++++++------- CMakeLists.txt | 12 ++++++------ cmake/executables.cmake | 2 +- doc/link_time_registration.md | 12 ++++++------ 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/BUILD_OPTIONS.md b/BUILD_OPTIONS.md index 83fe8ada6b..5ca243fc2c 100644 --- a/BUILD_OPTIONS.md +++ b/BUILD_OPTIONS.md @@ -26,13 +26,12 @@ For a more complete list of cmake variables, take a look in the file `CMakeCache -DBits="CosmoBit;DarkBit" # typical cosmology project -# Register migrated Bits' module functors with the Core at link -# time instead of compiling their rollcall headers into the Core: -# LINK_TIME_REGISTRATION (On|Off, default Off) -# Prototype; currently covers ExampleBit_A only. Editing a migrated -# Bit's rollcall header then recompiles only that Bit's objects plus -# a relink, rather than the Core's largest translation units. -# See doc/link_time_registration.md. +# Register Bits' module functors and the backend functors with the +# Core at link time instead of compiling their rollcall headers into +# the Core: LINK_TIME_REGISTRATION (On|Off, default Off) +# Editing a Bit's rollcall header then recompiles only that Bit's +# objects plus a relink, rather than the Core's largest translation +# units. See doc/link_time_registration.md. -DLINK_TIME_REGISTRATION=On diff --git a/CMakeLists.txt b/CMakeLists.txt index a04c8e2504..f9fad61601 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -610,18 +610,18 @@ include(cmake/externals.cmake) string (REPLACE ";" "," itch_with_commas "${itch}") -# Prototype: link-time (self-registering) module registration. Migrated Bits compile -# their own in-core rollcall macro expansions into a per-Bit registration translation +# Link-time (self-registering) module registration. Bits compile their own +# in-core rollcall macro expansions into a per-Bit registration translation # unit (/registration/_registration.cpp) that is linked only into the gambit # executable, instead of having their rollcall headers #included into the Core via the -# generated module_rollcall.hpp. Editing a migrated Bit's rollcall header then +# generated module_rollcall.hpp. Editing a Bit's rollcall header then # recompiles only that Bit's objects (plus a relink), not the Core. -option(LINK_TIME_REGISTRATION "Register migrated Bits' module functors and the backend functors with the Core at link time instead of compiling their rollcall headers into the Core" OFF) +option(LINK_TIME_REGISTRATION "Register Bits' module functors and the backend functors with the Core at link time instead of compiling their rollcall headers into the Core" OFF) set(LINK_TIME_REGISTRATION_BITS "") set(LINK_TIME_REGISTRATION_COMPONENTS "") if(LINK_TIME_REGISTRATION) - # Bits migrated to link-time registration so far (now: all of them). A Bit can - # only be listed here if it has a registration translation unit at + # Bits using link-time registration. A Bit can only be listed here if it has + # a registration translation unit at # /registration/_registration.cpp. foreach(bit ColliderBit CosmoBit DarkBit DecayBit ExampleBit_A ExampleBit_B FlavBit NeutrinoBit ObjectivesBit PrecisionBit SpecBit) if(";${GAMBIT_BITS};" MATCHES ";${bit};") diff --git a/cmake/executables.cmake b/cmake/executables.cmake index db1da7476d..d9c2a5ab3b 100644 --- a/cmake/executables.cmake +++ b/cmake/executables.cmake @@ -51,7 +51,7 @@ if(EXISTS "${PROJECT_SOURCE_DIR}/Core/") set(gambit_XTRA ${gambit_XTRA} ${YODA_LDFLAGS}) endif() # Registration translation units for components using link-time registration - # (migrated Bits, and the backends as a whole). These compile the in-core + # (the Bits, and the backends as a whole). These compile the in-core # expansion of each component's rollcall header(s), and are linked only into # the gambit executable: standalone executables get equivalent functor # definitions from their own main translation unit (via standalone_module.hpp), diff --git a/doc/link_time_registration.md b/doc/link_time_registration.md index 9c0503f028..34c30f453a 100644 --- a/doc/link_time_registration.md +++ b/doc/link_time_registration.md @@ -1,4 +1,4 @@ -# Link-time (self-registering) module registration — prototype +# Link-time (self-registering) module registration CMake option: `-DLINK_TIME_REGISTRATION=On` (default `Off`). Migrated: **all Bits** (ColliderBit, CosmoBit, DarkBit, DecayBit, ExampleBit_A, @@ -75,7 +75,7 @@ the dependency resolver, the likelihood container and the diagnostics all operat purely on the runtime registries. There is no need for a new registry or registrar class: the entire fix is to *relocate the expansion into a TU owned by the module*. -## What the prototype does +## What this does With `-DLINK_TIME_REGISTRATION=On`: @@ -201,9 +201,9 @@ randomness, per-point runtime estimates, and printer-ID assignment order own TU — cosmetic; see Caveats). `make ExampleBit_A_standalone` builds and runs successfully in both configurations. -## ColliderBit migration (second Bit, first non-trivial one) +## ColliderBit migration -ColliderBit was migrated as the stress test: its rollcall is a tree of five +ColliderBit was the stress test of this approach: its rollcall is a tree of five sub-rollcall headers, two of which are *generated* at build time by `collider_harvester.py` (`ColliderBit_models_rollcall.hpp`, `Py8Collider_typedefs.hpp`); it is dense with conditional-compilation guards @@ -287,7 +287,7 @@ by `gambit.hpp`'s include order (backends before modules); with separate registration TUs the order is link-order, and the binary aborted pre-main ("The backend "Pythia" is not known to GAMBIT"). -Following the prototype design rule (record passively, defer real work), +Following the design rule used throughout (record passively, defer real work), `set_classload_requirements` now applies the requirement immediately when the backend's version information is already available, and otherwise queues it; `backend_info::link_versions` retries the queue every time any backend @@ -326,7 +326,7 @@ Things that stay central (deliberately, for now): functors (primary model parameter functors) use additional registration calls (`register_model_functor_core`, claw bookkeeping) but follow the same self-registering pattern, so the same relocation should work for - `Models/models/*.hpp` if wanted; it is out of scope of this prototype. + `Models/models/*.hpp` if wanted; it has not yet been done. - **Backends**: `backend_rollcall.hpp` likewise still compiles into the Core. Backend functors are registered through the same kind of static-init calls (`register_backend_functor` → `Core().registerBackendFunctor`), so the pattern From 71f32f55eab8399e2c48eb94264305def10a6523 Mon Sep 17 00:00:00 2001 From: ChrisJChang Date: Fri, 7 Aug 2026 11:48:16 +0200 Subject: [PATCH 17/17] Remove unneccessary doc files --- doc/link_time_registration.md | 359 ---------------------- doc/link_time_registration_logs/README.md | 70 ----- 2 files changed, 429 deletions(-) delete mode 100644 doc/link_time_registration.md delete mode 100644 doc/link_time_registration_logs/README.md diff --git a/doc/link_time_registration.md b/doc/link_time_registration.md deleted file mode 100644 index 34c30f453a..0000000000 --- a/doc/link_time_registration.md +++ /dev/null @@ -1,359 +0,0 @@ -# Link-time (self-registering) module registration - -CMake option: `-DLINK_TIME_REGISTRATION=On` (default `Off`). Migrated: -**all Bits** (ColliderBit, CosmoBit, DarkBit, DecayBit, ExampleBit_A, -ExampleBit_B, FlavBit, NeutrinoBit, ObjectivesBit, PrecisionBit, SpecBit) and -the **Backends** (all frontend headers). With the option ON, -`module_rollcall.hpp` contains no module rollcall headers at all and -`gambit.hpp` no longer includes `backend_rollcall.hpp`; `gambit.cpp` expands -only the model rollcall. Only the **Models** remain on the legacy -compile-time path; the two mechanisms coexist (the option defaults to OFF, and -partially-migrated configurations work, as the incremental history of this -branch demonstrates). - -## The problem - -Every Bit's rollcall header is `#include`d into the Core by the generated -`Core/include/gambit/Core/module_rollcall.hpp`, whose only consumer is -`Core/src/gambit.cpp` (via `gambit.hpp`). The rollcall macros expand there in the -"in-core" context (`Elements/include/gambit/Elements/module_macros_incore_defs.hpp`), -producing every module functor *definition* plus its registration calls — all in one -enormous translation unit. Consequently, editing any Bit's rollcall header recompiles -`gambit.cpp` (the largest TU in the Core) and relinks everything. - -## What the in-core macros actually do - -For a single module function (worked example: `ExampleBit_A::nevents_pred`, -capability `nevents`, type `double`, one `DEPENDENCY(xsection, double)`), the in-core -expansion performs: - -`START_MODULE` (once per module), in `namespace Gambit::ExampleBit_A`: - -- Defines `ExampleBit_A_error()` / `ExampleBit_A_warning()` accessors - (function-local statics) and namespace-scope references that force their creation. -- `register_module_with_log("ExampleBit_A")` — adds a log tag via the - `Logging::tag2str()` / `Logging::components()` function-local-static registries. -- `register_module("ExampleBit_A", )` → `Core().registerModule(...)` - (module name + citation key list, used by diagnostics and the dependency resolver). -- A `Utils::python_interpreter_guard` global (keeps pybind11 alive through static init). -- Generic (fallback) `resolve_dependency` / `resolve_backendreq` / - `rt_register_*` function templates, later specialised per dependency/requirement. - -`START_CAPABILITY` / `DECLARE_FUNCTION`: - -- Declares tag structs `Gambit::Tags::nevents`, `Gambit::Tags::nevents_pred` - (incomplete types used only as template arguments, TU-local). -- Prototypes `void nevents_pred(double&)` (defined in the module's own sources). -- Defines the functor global `Functown::nevents_pred` of type - `module_functor` (or `model_functor` for `ModelParameters`); its - constructor takes `Models::ModelDB()` (the claw singleton) by reference. -- Defines the `Pipes::nevents_pred` globals: the `Param` safe-parameter map, - `ModelInUse` function pointer, `runOptions`, `Downstream::dependees`/`subcaps` - safe pointers, and (for loop managers) the `Loop` pipes. -- `register_function(...)` — connects those pipes to the functor's internals. -- `register_module_functor_core(...)` → `Core().registerModuleFunctor(...)`. - -`DEPENDENCY` / `NEEDS_MANAGER` / `ALLOW_MODEL(S)` / `BACKEND_REQ` / etc.: - -- Define a `dep_bucket` / `BE*_bucket` safety bucket in `Pipes::::Dep` - (or `::BEreq`), an explicit specialisation of `resolve_dependency` - / `resolve_backendreq` that downcasts the resolving functor and initialises the - bucket, and a `register_*` call that stores capability/type strings and the - resolver function pointer in the functor object. `NEEDS_MANAGER` additionally - calls `register_management_req(...)` → `Core().registerNestedModuleFunctor(...)`. - -### The key observation - -Every one of these side effects is either (a) a definition local to the expanding -TU, or (b) a call on a *function-local-static* singleton (`Core()`, -`Models::ModelDB()`, `Backends::backendInfo()`, the logging registries) made by a -namespace-scope `const int ... = register_*(...)` initialiser. In other words, -**GAMBIT's in-core macros are already a self-registration system, safe against the -static-initialisation-order fiasco**. The Core does not consume any compile-time -knowledge from the rollcall headers other than these expansions; `gambit.cpp`, -the dependency resolver, the likelihood container and the diagnostics all operate -purely on the runtime registries. There is no need for a new registry or registrar -class: the entire fix is to *relocate the expansion into a TU owned by the module*. - -## What this does - -With `-DLINK_TIME_REGISTRATION=On`: - -1. **Harvester** (`module_harvester.py -r ExampleBit_A`, driven from the top-level - `CMakeLists.txt` via `MODULE_HARVESTER_EXTRA_ARGS`): ExampleBit_A is harvested - exactly as before — its functor types still enter `module_functor_types.hpp` - (so the central explicit template instantiations in `Core/src/functors.cpp` - still cover it), its types header still enters `module_types_rollcall.hpp`, - it still appears in `config/gambit_bits.yaml` and the standalone type pickles — - but its rollcall header is **omitted from the include list** in - `module_rollcall.hpp`. The Core no longer compiles anything from ExampleBit_A. - -2. **Registration TU** (`ExampleBit_A/registration/ExampleBit_A_registration.cpp`): - guarded by `#ifdef LINK_TIME_REGISTRATION`, it does precisely what - `module_rollcall.hpp` used to do for this Bit: - - ```cpp - #include "gambit/Elements/module_macros_incore.hpp" - #include "gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp" - ``` - - `cmake/executables.cmake` compiles this file **into the `gambit` executable - only** (one entry per migrated Bit in `LINK_TIME_REGISTRATION_BITS`). It must - not join the Bit's OBJECT library, because standalone executables - (`make ExampleBit_A_standalone`) link those same objects together with their - own in-core expansion compiled from the standalone main via - `standalone_module.hpp` (with `STANDALONE` defined) — adding the registration - TU there would produce duplicate definitions of every `Functown::` functor. - Because the object file is passed directly to the linker (no archive), no - dead-stripping or hidden-visibility issue arises; registration runs pre-main, - single-threaded, as before. - -3. **Determinism fix** (independent benefit, also applied to the legacy path): - the generated headers (`module_types_rollcall.hpp`, `module_functor_types.hpp`, - `backend_functor_types.hpp`, standalone functor lists) were emitted in Python - set iteration order, which changes from run to run, and `module_rollcall.hpp` - was rewritten unconditionally on every harvest. Both meant that *any* harvester - re-run dirtied headers included by every TU in the tree, masking incremental - builds. All generated lists are now sorted and `module_rollcall.hpp` is only - rewritten when its content changes. - -### Rebuild scope after the change - -Touching `ExampleBit_A/include/gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp` now -recompiles: - -- the module harvester re-run (output unchanged → no generated headers dirtied), -- `ExampleBit_A`'s own objects that include the rollcall header - (`src/ExampleBit_A.cpp` in the in-module context), -- the registration TU, -- one link of `gambit`. - -On the legacy path the same touch additionally recompiles `Core/src/gambit.cpp` -(the in-core expansion of *all* rollcall headers). See "Measurements" below. - -## Measurements - -Build configuration: `-DBits="ExampleBit_A;ExampleBit_B" -DWITH_MPI=Off --DCMAKE_BUILD_TYPE=Release`, GCC 13.3, 4 cores. Experiment (after a full build -and a no-op `make gambit` showing 0 recompilations): -`touch ExampleBit_A/include/gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp && time make gambit`. - -| | legacy (`OFF`) | link-time (`ON`) | -|---|---|---| -| objects recompiled | `ExampleBit_A.cpp.o`, **`Core/src/gambit.cpp.o`**, link | `ExampleBit_A.cpp.o`, `ExampleBit_A_registration.cpp.o`, link | -| wall time | 1m50.4s | 0m23.6s | - -Raw logs: `doc/link_time_registration_logs/`. The Core's largest TU no longer -rebuilds; the gap grows with the number/size of Bits in the build (this -configuration contains only the two small ExampleBits — in a full build, -`gambit.cpp` expands every Bit's rollcall header). - -With ColliderBit also in the build (`-DBits="ColliderBit;ExampleBit_A;ExampleBit_B" --DWITH_HEPMC=ON -DWITH_YODA=OFF`, same machine), repeating the experiments: - -| touch | legacy (`OFF`) | link-time (`ON`) | -|---|---|---| -| `ColliderBit_rollcall.hpp` | 23 ColliderBit TUs + **`gambit.cpp`** + link, 9m22s | 23 ColliderBit TUs + `ColliderBit_registration.cpp` + link, 7m04s | -| `ExampleBit_A_rollcall.hpp` | `ExampleBit_A.cpp` + **`gambit.cpp`** + link, 4m57s | `ExampleBit_A.cpp` + `ExampleBit_A_registration.cpp` + link, 0m31s | - -Two observations. First, the cross-Bit isolation is the headline: once a big Bit -is in the build, the legacy path makes *every* Bit's rollcall edit pay the -full `gambit.cpp` recompile (ExampleBit_A: 4m57s → 31s, ~10x). Second, for the -big Bit itself most of the remaining cost is its own 23 source files, which -include the rollcall header and rebuild on either path — reducing that is a -module-internal layout question (e.g. splitting rollcall includes), orthogonal -to Core coupling. - -With the backends also migrated, touching one frontend header -(`Backends/include/gambit/Backends/frontends/LibFirst_1_0.hpp`): - -| | legacy (`OFF`) | link-time (`ON`) | -|---|---|---| -| objects recompiled | **`gambit.cpp`** + link | `Backends_registration.cpp` + link | -| wall time | 4m32s | 0m55s | - -(The frontend's own `.cpp`, when enabled in the configuration, rebuilds on -either path.) - -With **all Bits** in the build (no `-DBits` restriction, `-DBUILD_FS_MODELS=None`, -`-DWITH_HEPMC=ON -DWITH_YODA=OFF`, same machine; 1654 module functors and 1610 -backend functors registered): - -| touch | legacy (`OFF`) | link-time (`ON`) | -|---|---|---| -| `DarkBit_rollcall.hpp` | 30 DarkBit TUs + **`gambit.cpp`** + link, 33m12s | 30 DarkBit TUs + `DarkBit_registration.cpp` + link, 9m47s | -| `ExampleBit_A_rollcall.hpp` | `ExampleBit_A.cpp` + **`gambit.cpp`** + link, 25m41s | `ExampleBit_A.cpp` + `ExampleBit_A_registration.cpp` + link, 0m49s | - -In the full configuration the `gambit.cpp` recompile alone costs roughly -twenty-five minutes on this machine, and the legacy path pays it for *every* -rollcall edit in *any* Bit: a one-line change in the smallest Bit goes from -25m41s to 49s (~31x). - -Runtime equivalence (same configuration, `spartan.yaml` with the built-in -`random` scanner standing in for the external `diver`): both configurations -register exactly 240 module functors and 84 backend functors, the logged -masterGraph functor table (origin, function, capability, type, status, #deps, -#backend-reqs) is byte-identical, and the dependency-resolution log content -(candidate vertices, applied rules, evaluation order) is identical after -stripping timestamps. Remaining log differences are unseeded scanner -randomness, per-point runtime estimates, and printer-ID assignment order -(registry iteration order shifts because ExampleBit_A now registers from its -own TU — cosmetic; see Caveats). `make ExampleBit_A_standalone` builds and -runs successfully in both configurations. - -## ColliderBit migration - -ColliderBit was the stress test of this approach: its rollcall is a tree of five -sub-rollcall headers, two of which are *generated* at build time by -`collider_harvester.py` (`ColliderBit_models_rollcall.hpp`, -`Py8Collider_typedefs.hpp`); it is dense with conditional-compilation guards -(`HAVE_PYBIND11`, `EXCLUDE_HEPMC`, `EXCLUDE_YODA`); it uses BOSSed Pythia types -in backend requirements, loop-managed event-loop functors, model groups, and -`NEEDS_CLASSES_FROM`. Findings: - -- **One real hidden coupling found and fixed**: the in-core expansions of - `NEEDS_CLASSES_FROM` (→ `set_classload_requirements`) and - `ACTIVATE_BACKEND_REQ_FOR_MODELS` (→ `set_backend_rule_for_model`) call - functions declared in `Backends/ini_functions.hpp`, which the in-core macro - header never included. The legacy path only compiles because `gambit.hpp` - happens to include `backend_rollcall.hpp` *before* `module_rollcall.hpp`. - `module_macros_incore_defs.hpp` now includes the declarations it uses, making - the in-core context self-contained. No other order-dependent declaration was - hit by the full ColliderBit rollcall tree. -- **Config-guard consistency is automatic**: `HAVE_PYBIND11`, `EXCLUDE_HEPMC` - and `EXCLUDE_YODA` all come from the generated `cmake_variables.hpp`, so the - registration TU, the module objects and the (legacy) Core expansion always - agree on which rollcall sections exist. -- **Generated sub-headers need no special handling**: the registration TU is a - source of the `gambit` target, which depends on the ColliderBit object - library, which depends on `collider_harvest` — the same transitive ordering - that protects `module_harvest` today. -- The migration itself was exactly the advertised recipe: one two-include - registration TU plus one entry in `LINK_TIME_REGISTRATION_BITS`. - -### Validation environment caveat - -The validation build used `-DWITH_YODA=OFF` (a new, explicit opt-out added in -this branch — the YODA tarball host is unreachable from the build sandbox; -HepMC3 3.2.5 was supplied as the authentic md5-verified tarball). YODA-guarded -measurement functions are therefore compiled out *identically in both -configurations*, so the legacy-vs-link-time comparison is unaffected, but the -`EXCLUDE_YODA=0` sections of the measurements rollcall have not been exercised -under link-time registration. They use the same macros as the rest of the tree -(no new macro kinds), so no new failure mode is expected. CBS -(`ColliderBit Solo`) does not build in this YODA-less configuration in *either* -mode — its main source unconditionally references the Rivet/Contur/nulike -frontends, which the configuration excludes — verified to fail identically -(same first error in `solo.cpp`) with the option OFF and ON, i.e. a property -of the configuration, not of link-time registration. CBS links the ColliderBit -object library plus its own in-core expansion from `standalone_module.hpp`, -neither of which this change touches. - -Runtime equivalence with ColliderBit migrated: both configurations register -exactly 463 module and 135 backend functors; the masterGraph functor table -(464 rows, 214 of them ColliderBit) is byte-identical; the dependency -resolution log differs only in timestamps, printer-ID assignment order and -unseeded scanner output, as before. - -## Backends migration - -The backends were migrated as a third step. They do not fit the per-Bit CMake -pattern, so the wiring differs: - -- `backend_rollcall.hpp` (the list of all frontend headers) is generated by the - *backend* harvester and `#include`d directly by `gambit.hpp`. No harvester - change is involved: when `LINK_TIME_REGISTRATION` is ON, `gambit.hpp` simply - skips the include via the global compile definition, and a single - registration TU (`Backends/registration/Backends_registration.cpp`, again - linked only into the `gambit` executable) expands it instead. `gambit.cpp` - itself has no compile-time dependency on any backend declaration. -- Granularity is the whole Backends directory: the frontend list is dynamic - (harvested, with config-dependent exclusions), so per-frontend TUs would - need code generation. Editing one frontend header therefore recompiles the - one registration TU (which includes *all* frontend headers) plus the - frontend's own source file — still a fraction of a `gambit.cpp` compile. -- `backend_macros.hpp` already includes `functor_definitions.hpp`, so the - registration TU instantiates the functor templates it needs locally. - -### The static-initialisation-order bug this exposed (and its fix) - -This migration hit the predicted cross-TU initialisation-order hazard for -real: the in-core expansion of `NEEDS_CLASSES_FROM(Pythia, default)` (in -ColliderBit's registration TU) calls `set_classload_requirements`, which -translated version strings via `backendInfo().version_from_safe_version()` — -maps that are only populated once the Pythia frontend's `LOAD_LIBRARY` -registration has run. In the legacy single-TU world the order was guaranteed -by `gambit.hpp`'s include order (backends before modules); with separate -registration TUs the order is link-order, and the binary aborted pre-main -("The backend "Pythia" is not known to GAMBIT"). - -Following the design rule used throughout (record passively, defer real work), -`set_classload_requirements` now applies the requirement immediately when the -backend's version information is already available, and otherwise queues it; -`backend_info::link_versions` retries the queue every time any backend -registers a version. Registration is thereby order-independent *by -construction* — no reliance on link order. A requirement that is still -unfulfilled after static initialisation (i.e. the backend never registered at -all — a configuration that previously *terminated pre-main* in the legacy -path) is now reported as a proper error by -`check_deferred_classload_requirements()`, called from -`gambit_core::accountForMissingClasses()`. Standalone executables still -register backends before modules within their single TU, so they take the -immediate path and behave exactly as before. - -This was the only order-sensitive registration step found: all other -module-side registrations store strings/function pointers in the functor -itself, and all backend-side registrations only touch `backendInfo()`/`Core()` -singletons. - -## What remains to migrate a real Bit - -Per Bit, the migration recipe is mechanical: - -1. Create `/registration/_registration.cpp` (two includes, as above). -2. Add the Bit to `LINK_TIME_REGISTRATION_BITS` in the top-level `CMakeLists.txt`. - -Things that stay central (deliberately, for now): - -- **Explicit functor template instantiations**: `module_functor` member - definitions live in `functor_definitions.hpp` and are instantiated centrally in - `Core/src/functors.cpp` from the harvested `module_functor_types.hpp`. A migrated - Bit introducing a *new* return type still dirties that one list (one Core TU - recompiles — much smaller than `gambit.cpp`). A follow-up could instead include - `functor_definitions.hpp` in each registration TU and harvest a reduced central - list, making type additions module-local too. -- **Models**: `model_rollcall.hpp` is still compiled into the Core. Model-module - functors (primary model parameter functors) use additional registration calls - (`register_model_functor_core`, claw bookkeeping) but follow the same - self-registering pattern, so the same relocation should work for - `Models/models/*.hpp` if wanted; it has not yet been done. -- **Backends**: `backend_rollcall.hpp` likewise still compiles into the Core. - Backend functors are registered through the same kind of static-init calls - (`register_backend_functor` → `Core().registerBackendFunctor`), so the pattern - extends, but the backend macro machinery (BOSS, classloading) was not audited. -- **Harvesters/GUM/printers**: unaffected. The `-r` option changes only the - include list of `module_rollcall.hpp`; all other harvester outputs are - byte-identical, and the printer harvester does not read rollcall headers. - -### Caveats / behavioural differences found - -- **Registration order across TUs**: within one TU, registration order follows the - rollcall header top-to-bottom, as before. *Across* modules the order is now - unspecified (link order in practice) instead of `module_rollcall.hpp` include - order. Nothing in the Core depends on registration order (registries are - containers keyed/sorted downstream), but iteration order over - `Core().getModuleFunctors()` shifts: observed as different (but internally - consistent) printer-ID assignments in the logs. Scan output labels and values - are unaffected. -- **One-definition headers**: `Utils/static_members.hpp` *defines* static data - members and is pulled in by the in-core macro header, so a registration TU - would duplicate them against the main TU. It now honours - `GAMBIT_NO_STATIC_MEMBER_DEFINITIONS`, which registration TUs define. Any - future header that defines objects from the in-core context would need the - same treatment (this was the only one in the current tree). -- **Standalones**: unchanged by construction (verified by building - `ExampleBit_A_standalone` in both configurations). -- **`QUICK_FUNCTION`-style ad-hoc declarations in the Core** would be a blocker if - any Core source declared extra functions for a migrated Bit at compile time; - none do for ExampleBit_A (or any other Bit; the pattern only appears in - standalone mains, which keep their own expansion). diff --git a/doc/link_time_registration_logs/README.md b/doc/link_time_registration_logs/README.md deleted file mode 100644 index 1a17ab78e0..0000000000 --- a/doc/link_time_registration_logs/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Touch-test evidence logs - -Experiment, run in both configurations after a successful full build and a no-op -`make gambit` (0 recompilations): - - touch ExampleBit_A/include/gambit/ExampleBit_A/ExampleBit_A_rollcall.hpp - time make gambit - -Configuration: `-DBits="ExampleBit_A;ExampleBit_B" -DWITH_MPI=Off --DCMAKE_BUILD_TYPE=Release`, GCC 13.3.0, 4 cores, cmake 3.28 (Makefiles). - -| configuration | objects recompiled | wall time | -|---|---|---| -| `LINK_TIME_REGISTRATION=Off` (`touch_test_legacy.log`) | `ExampleBit_A/src/ExampleBit_A.cpp.o`, `Core/src/gambit.cpp.o`, link | 1m50.4s | -| `LINK_TIME_REGISTRATION=On` (`touch_test_ltr.log`) | `ExampleBit_A/src/ExampleBit_A.cpp.o`, `ExampleBit_A/registration/ExampleBit_A_registration.cpp.o`, link | 0m23.6s | - -In the legacy configuration the Core's largest translation unit -(`Core/src/gambit.cpp`, which #includes every Bit's rollcall header via the -generated `module_rollcall.hpp`) rebuilds on every rollcall edit. With -link-time registration it does not; only the Bit's own objects rebuild, and -the difference grows with the number and size of Bits in the build (this -measurement used the two small ExampleBits only). - -Both configurations were measured with the harvested-header determinism fix -in place (sorted generation + write-only-if-changed for module_rollcall.hpp). -Without that fix, *both* configurations intermittently rebuild near-everything -after any harvester re-run, because the generated type-header include order -was randomised by Python set iteration. - -## ColliderBit round - -Configuration: `-DBits="ColliderBit;ExampleBit_A;ExampleBit_B" -DWITH_HEPMC=ON --DWITH_YODA=OFF -DWITH_MPI=Off -DCMAKE_BUILD_TYPE=Release`, same machine. -Same protocol (full build, no-op `make gambit` showing 0 recompiles, then touch + time). - -| touch | configuration | objects recompiled | wall time | log | -|---|---|---|---|---| -| `ColliderBit_rollcall.hpp` | legacy | 23 ColliderBit TUs + `gambit.cpp.o` + link | 9m22.5s | `touch_cb_legacy.log` | -| `ColliderBit_rollcall.hpp` | link-time | 23 ColliderBit TUs + `ColliderBit_registration.cpp.o` + link | 7m04.1s | `touch_cb_ltr.log` | -| `ExampleBit_A_rollcall.hpp` | legacy | `ExampleBit_A.cpp.o` + `gambit.cpp.o` + link | 4m56.5s | `touch_eba_legacy.log` | -| `ExampleBit_A_rollcall.hpp` | link-time | `ExampleBit_A.cpp.o` + `ExampleBit_A_registration.cpp.o` + link | 0m31.0s | `touch_eba_ltr.log` | - -The ExampleBit_A rows show the cross-Bit isolation effect: in the legacy path, -adding a large Bit to the build makes every other Bit's rollcall edits pay that -Bit's share of the `gambit.cpp` recompile. - -## Backends round - -Same configuration and protocol; experiment: -`touch Backends/include/gambit/Backends/frontends/LibFirst_1_0.hpp && time make gambit`. - -| configuration | objects recompiled | wall time | log | -|---|---|---|---| -| legacy | `gambit.cpp.o` + link | 4m32.2s | `touch_be_legacy.log` | -| link-time | `Backends_registration.cpp.o` + link | 0m55.2s | `touch_be_ltr.log` | - -## All-Bits round - -Configuration: all Bits (no `-DBits` restriction), `-DBUILD_FS_MODELS=None --DWITH_HEPMC=ON -DWITH_YODA=OFF -DWITH_MPI=Off -DCMAKE_BUILD_TYPE=Release`, -same machine. 1654 module functors + 1610 backend functors registered; -module and backend functor tables byte-identical between the two -configurations, spartan smoke test passing in both. - -| touch | configuration | objects recompiled | wall time | log | -|---|---|---|---|---| -| `DarkBit_rollcall.hpp` | legacy | 30 DarkBit TUs + `gambit.cpp.o` + link | 33m11.9s | `touch_db_legacy.log` | -| `DarkBit_rollcall.hpp` | link-time | 30 DarkBit TUs + `DarkBit_registration.cpp.o` + link | 9m47.1s | `touch_db_ltr.log` | -| `ExampleBit_A_rollcall.hpp` | legacy | `ExampleBit_A.cpp.o` + `gambit.cpp.o` + link | 25m41.4s | `touch_eba_legacy_all.log` | -| `ExampleBit_A_rollcall.hpp` | link-time | `ExampleBit_A.cpp.o` + `ExampleBit_A_registration.cpp.o` + link | 0m49.0s | `touch_eba_ltr_all.log` |