From e12a98fb9a4eebf7c079c152388bad14a836d529 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 13:04:38 -0400 Subject: [PATCH 1/6] Bind infinite parameter values in reform payloads PolicyEngine parameters take infinite values (an uncapped threshold is .inf), and the Budget dashboard's salary_sacrifice_cap measure carries one, but JSON cannot hold them, so its reform could not be receipted. Reform payload digests now encode inf as {"$nonfinite": "Infinity"} and -inf as {"$nonfinite": "-Infinity"}; a payload using the key $nonfinite itself is refused, so no two payloads share a digest. NaN stays refused. Scenario parameter changes are converted once, at the digest. A payload without infinities digests exactly as before (differential property test against the previous definition). Co-Authored-By: Claude Opus 5.5 --- docs/receipts.md | 4 +- .../schemas/execution_receipt.v1.schema.json | 2 +- src/policyengine/provenance/receipt.py | 48 +++++++++-- tests/test_execution_receipt.py | 84 +++++++++++++++++++ 4 files changed, 129 insertions(+), 9 deletions(-) diff --git a/docs/receipts.md b/docs/receipts.md index 944ef877..d0f17166 100644 --- a/docs/receipts.md +++ b/docs/receipts.md @@ -130,7 +130,7 @@ const dropNulls = (v) => const digest = (value) => createHash("sha256").update(canonicalize(dropNulls(value)), "utf8").digest("hex"); ``` -Reform digests are the exception: a reform payload is digested with its nulls kept, because there a null can mean something. `Scenario.from_reform({"p": {"2027": None}})` removes a value that `{"p": {}}` keeps, so the two must not share a digest. +Reform digests are the exception: a reform payload is digested with its nulls kept, because there a null can mean something. `Scenario.from_reform({"p": {"2027": None}})` removes a value that `{"p": {}}` keeps, so the two must not share a digest. Reform payloads may also hold infinite numbers, which PolicyEngine parameters use for an uncapped threshold and JSON cannot hold. `inf` is digested as the object `{"$nonfinite": "Infinity"}` and `-inf` as `{"$nonfinite": "-Infinity"}`, and a payload that uses the key `$nonfinite` itself is refused, so no two payloads share a digest. NaN is refused, and a payload without infinities digests as before. The pre-promotion replay interface with policyengine-scorecard follows a different rule, set by its own schemas: plain RFC 8785 over the document as sent, nulls included. `policyengine.provenance.canonical.canonical_sha256` computes that. @@ -139,7 +139,7 @@ The pre-promotion replay interface with policyengine-scorecard follows a differe | `execution_identity_sha256` | `schema_version`, `country_id`, `wrapper`, `model`, `core`, `data`, `certification_status`, and the certificate's content: bundle version, record digest, basis, certified versions, dataset digest and certifier. Results with the same execution identity ran on the same runtime and data under the same certificate, however the receipt was built. | | `receipt_sha256` | The whole receipt except `runtime` and the reform and baseline `name` labels. Renaming a measure therefore does not change it. | | `certificate.certificate_sha256` | The bundle manifest's `data_releases.uk` record. | -| `reform.reform_sha256` | `{"representation": ..., "payload": ...}`, with the payload exactly as given, nulls kept (strict JSON conversion, then RFC 8785, with no null removal). For a composition the payload is `{"applied_before_data_load": ..., "application": "parameter_changes_then_modifiers", "members": [{"representation", "reform_sha256", "binding"}, ...]}`, with members in application order. | +| `reform.reform_sha256` | `{"representation": ..., "payload": ...}`, with the payload exactly as given, nulls kept (strict JSON conversion with infinities as `{"$nonfinite": "Infinity"}` or `"-Infinity"`, then RFC 8785, with no null removal). For a composition the payload is `{"applied_before_data_load": ..., "application": "parameter_changes_then_modifiers", "members": [{"representation", "reform_sha256", "binding"}, ...]}`, with members in application order. | | `outputs_sha256` | The output payload. | Reform digests are never normalized across representations. The same keys can legitimately mean different things in different representations: for example, a bare year in a policyengine-uk country dictionary changes that one year only. So each digest is tied to the representation that was submitted. The v1 representations are `policyengine.parameter_values/v1`, `policyengine.reform_dict/v1`, `policyengine-uk.reform_dict/v1`, `policyengine-uk.scenario/v1`, `policyengine-uk.scenario_composition/v1` and `policyengine-sim-api.policy/v1`. diff --git a/src/policyengine/data/schemas/execution_receipt.v1.schema.json b/src/policyengine/data/schemas/execution_receipt.v1.schema.json index 31cb849a..d6cd2f96 100644 --- a/src/policyengine/data/schemas/execution_receipt.v1.schema.json +++ b/src/policyengine/data/schemas/execution_receipt.v1.schema.json @@ -644,7 +644,7 @@ "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "outputs_sha256": "the canonical output payload the receipt binds", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", - "reform.reform_sha256": "RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order" + "reform.reform_sha256": "RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {\"$nonfinite\": \"Infinity\"} or {\"$nonfinite\": \"-Infinity\"}, and a payload may not use the key $nonfinite itself" }, "x-policyengine-rules": [ "years is sorted ascending with no duplicates, and each year is written as an integer (2027, not 2027.0).", diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index b40e17ce..7a085024 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -188,7 +188,7 @@ "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", "certificate.certificate_sha256": "the bundle manifest's data_releases. record", - "reform.reform_sha256": "RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order", + "reform.reform_sha256": 'RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {"$nonfinite": "Infinity"} or {"$nonfinite": "-Infinity"}, and a payload may not use the key $nonfinite itself', "outputs_sha256": "the canonical output payload the receipt binds", } @@ -661,17 +661,54 @@ def content_sha256(value: Any) -> str: raise ReceiptError(str(exc)) from exc +#: The reserved member that stands for an infinite number in a reform payload. +NONFINITE_KEY = "$nonfinite" + + +def _with_infinities_encoded(value: Any, path: str = "$") -> Any: + """A reform payload with each infinite number replaced by a reserved object. + + PolicyEngine parameters take infinite values (an uncapped threshold is + ``.inf``), which JSON cannot hold. ``inf`` becomes ``{"$nonfinite": + "Infinity"}`` and ``-inf`` becomes ``{"$nonfinite": "-Infinity"}``. A payload + that uses the key ``$nonfinite`` itself is refused, so no other payload + encodes to the same JSON. NaN is left for :func:`strict_json_value` to + refuse: it has no one value to bind. + """ + + if isinstance(value, Mapping): + if any(key == NONFINITE_KEY for key in value): + raise ReceiptError( + f"{path}: the key {NONFINITE_KEY} is reserved for infinite numbers." + ) + return { + key: _with_infinities_encoded(item, f"{path}.{key}") + for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [ + _with_infinities_encoded(item, f"{path}[{index}]") + for index, item in enumerate(value) + ] + number = value.item() if _numpy_scalar(value) else value + if isinstance(number, float) and math.isinf(number): + return {NONFINITE_KEY: "Infinity" if number > 0 else "-Infinity"} + return value + + def _payload_sha256(value: Any) -> str: """Digest of a reform payload: strict JSON with nulls kept, RFC 8785, SHA-256. Unlike :func:`content_sha256`, nulls are not removed: in a reform a null can mean something (``Scenario.from_reform({"p": {"2027": None}})`` removes a value that ``{"p": {}}`` keeps), so the payload is digested - exactly as given. + exactly as given. Infinite numbers are encoded as reserved objects + (:func:`_with_infinities_encoded`); a payload without them digests as it + did before they were allowed. """ - converted = strict_json_value(value) try: + converted = strict_json_value(_with_infinities_encoded(value)) return canonical_sha256(converted) except RecursionError as exc: raise ReceiptError("The value is nested too deeply.") from exc @@ -1960,9 +1997,8 @@ def scenario_reform_identity( return ReformIdentity(kind="current_law", name=name) payload: dict[str, Any] = { "applied_before_data_load": applied_before_data_load, - "parameter_changes": strict_json_value(parameter_changes) - if parameter_changes - else None, + # Converted once, with infinities encoded, when the payload is digested. + "parameter_changes": parameter_changes if parameter_changes else None, "simulation_modifier": None, } binding: Literal["content", "code_revision"] = "content" diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 2106088b..ce4bca75 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -3604,3 +3604,87 @@ def test_is_against_a_call_other_than_type_is_refused(tmp_path): _identity(module.raise_rate) finally: sys.modules.pop(name, None) + + +# --- Infinite parameter values in reform payloads --------------------------------- + + +CAP = "gov.hmrc.pensions.salary_sacrifice_pension_cap" + + +def _reform_digest(payload): + return reform_identity(payload, "policyengine-uk.reform_dict/v1").reform_sha256 + + +def test_an_infinite_parameter_value_binds_by_its_reserved_encoding(): + """An uncapped threshold is ``inf`` in PolicyEngine; the digest names it.""" + + payload = {CAP: {"2026-04-06.2100-12-31": float("inf")}} + assert _reform_digest(payload) == canonical_sha256( + { + "representation": "policyengine-uk.reform_dict/v1", + "payload": {CAP: {"2026-04-06.2100-12-31": {"$nonfinite": "Infinity"}}}, + } + ) + + +def test_infinities_differ_from_each_other_and_from_any_finite_cap(): + values = [float("inf"), float("-inf"), 1e308, 2000.0, 0.0] + digests = {_reform_digest({CAP: {"2027": value}}) for value in values} + assert len(digests) == len(values) + + +def test_a_numpy_infinity_binds_as_a_float_one(): + numpy = pytest.importorskip("numpy") + for value in (numpy.float64("inf"), numpy.float32("-inf")): + assert _reform_digest({CAP: {"2027": value}}) == _reform_digest( + {CAP: {"2027": float(value)}} + ) + + +@pytest.mark.parametrize( + "payload", + [ + {CAP: {"2027": {"$nonfinite": "Infinity"}}}, + {"$nonfinite": "Infinity"}, + [{"$nonfinite": "-Infinity"}], + ], + ids=["value", "top-level", "array-member"], +) +def test_a_payload_using_the_reserved_key_is_refused(payload): + """Otherwise the object would digest like the infinity it spells.""" + + with pytest.raises(ReceiptError, match=r"\$nonfinite is reserved"): + _reform_digest(payload) + + +def test_nan_is_still_refused(): + with pytest.raises(ReceiptError, match="not a finite number"): + _reform_digest({CAP: {"2027": float("nan")}}) + + +def test_a_scenario_with_an_infinite_parameter_change_binds(): + identity = scenario_reform_identity( + _scenario(parameter_changes={CAP: {"2027": float("inf")}}) + ) + assert identity.binding == "content" + assert ( + identity.reform_sha256 + != scenario_reform_identity( + _scenario(parameter_changes={CAP: {"2027": 1e308}}) + ).reform_sha256 + ) + + +@settings(max_examples=200, deadline=None) +@given(payload=REFORM_PAYLOADS) +def test_payloads_without_infinities_digest_as_before(payload): + """Differential: allowing infinities changes no finite payload's digest.""" + + from policyengine.provenance.receipt import strict_json_value + + assert _reform_digest(payload) == canonical_sha256( + strict_json_value( + {"representation": "policyengine-uk.reform_dict/v1", "payload": payload} + ) + ) From cab895b370af12dc7dd9c42bd0ca937bc3da1f7d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 13:04:38 -0400 Subject: [PATCH 2/6] Add changelog fragment for #587 Co-Authored-By: Claude Opus 5.5 --- changelog.d/587.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/587.added.md diff --git a/changelog.d/587.added.md b/changelog.d/587.added.md new file mode 100644 index 00000000..5f9c364c --- /dev/null +++ b/changelog.d/587.added.md @@ -0,0 +1 @@ +Execution receipt reform digests accept infinite parameter values (an uncapped threshold is `.inf`), encoding `inf` and `-inf` as `{"$nonfinite": "Infinity"}` and `{"$nonfinite": "-Infinity"}`. A payload that uses the key `$nonfinite` itself is refused, and finite payloads digest as before. From dab8f45a76a7baad8567d856863920f815ae723a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 11:01:49 -0400 Subject: [PATCH 3/6] Encode infinities inside the strict walk; pin both spellings Round-1 review of #588: - M1: rebuilding mappings before strict conversion dropped one of two equal keys, so payloads refused before (colliding keys) were accepted. Infinities are now encoded inside _strict_json_value, after the collision check, and the reserved key is checked on the converted text. - M2: the -Infinity spelling is pinned by exact canonical bytes. - L1: a tuple case for the reserved key and for inf. - L3: keys convert to an exact str, so a str subclass's __eq__ cannot hide the reserved key or a collision. - L4: docs say what the encoding guarantees and what a JavaScript reform verifier must do. - L5: an infinite modifier default binds by float.hex(); NaN is refused. - L6: the differential test now generates nested payloads with int and colliding keys, tuples, nulls, strings, dates, numpy scalars and large integers. Co-Authored-By: Claude Opus 5.5 --- docs/receipts.md | 4 +- src/policyengine/provenance/receipt.py | 80 +++++------ tests/test_execution_receipt.py | 186 ++++++++++++++++++++++--- 3 files changed, 205 insertions(+), 65 deletions(-) diff --git a/docs/receipts.md b/docs/receipts.md index d0f17166..39c8c18a 100644 --- a/docs/receipts.md +++ b/docs/receipts.md @@ -50,7 +50,7 @@ There are two ways to bind a policyengine-uk `Scenario`: A scenario with a module-level `simulation_modifier` needs `code_revision={"repository": ..., "git_sha": }`. The identity then says `binding: "code_revision"` and binds: - the function's module and qualified name; - the SHA-256 of its source and of its module's source file; -- the default arguments of every function in that module, encoded with their types and floats by their bits, so `1` and `1.0`, `0.0` and `-0.0`, a tuple and a list, or `None` and a missing key never share a digest (defaults with no exact encoding, such as objects, are refused, and so are two defaults that are one mutable object, such as one list set as both `first` and `second`, since the encoding cannot tell that from two equal lists); +- the default arguments of every function in that module, encoded with their types and floats by their bits (`float.hex()`, which writes infinities as `inf` and `-inf`; NaN is refused), so `1` and `1.0`, `0.0` and `-0.0`, a tuple and a list, or `None` and a missing key never share a digest (defaults with no exact encoding, such as objects, are refused, and so are two defaults that are one mutable object, such as one list set as both `first` and `second`, since the encoding cannot tell that from two equal lists); - that revision. None of these depends on where the code is checked out or which Python version runs it, so CI, a laptop and the server compute the same digest for the same code. @@ -130,7 +130,7 @@ const dropNulls = (v) => const digest = (value) => createHash("sha256").update(canonicalize(dropNulls(value)), "utf8").digest("hex"); ``` -Reform digests are the exception: a reform payload is digested with its nulls kept, because there a null can mean something. `Scenario.from_reform({"p": {"2027": None}})` removes a value that `{"p": {}}` keeps, so the two must not share a digest. Reform payloads may also hold infinite numbers, which PolicyEngine parameters use for an uncapped threshold and JSON cannot hold. `inf` is digested as the object `{"$nonfinite": "Infinity"}` and `-inf` as `{"$nonfinite": "-Infinity"}`, and a payload that uses the key `$nonfinite` itself is refused, so no two payloads share a digest. NaN is refused, and a payload without infinities digests as before. +Reform digests are the exception: a reform payload is digested with its nulls kept, because there a null can mean something. `Scenario.from_reform({"p": {"2027": None}})` removes a value that `{"p": {}}` keeps, so the two must not share a digest. Reform payloads may also hold infinite numbers, which PolicyEngine parameters use for an uncapped threshold and JSON cannot hold. `inf` is digested as the object `{"$nonfinite": "Infinity"}` and `-inf` as `{"$nonfinite": "-Infinity"}`, and a payload that uses the key `$nonfinite` itself is refused, so no other payload digests like an infinity. The encoding happens inside strict conversion, so a payload it refuses without infinities, such as one with colliding keys, is refused with them too. NaN is refused, and a payload without infinities digests as before. The JavaScript above covers content digests only: a JavaScript verifier of reform digests must keep nulls, replace `Infinity` and `-Infinity` with the reserved objects before canonicalizing, and refuse a payload with a `$nonfinite` key. The pre-promotion replay interface with policyengine-scorecard follows a different rule, set by its own schemas: plain RFC 8785 over the document as sent, nulls included. `policyengine.provenance.canonical.canonical_sha256` computes that. diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index 7a085024..05e680f4 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -598,12 +598,26 @@ def strict_json_value(value: Any, path: str = "$") -> Any: """ try: - return _strict_json_value(value, path) + return _strict_json_value(value, path, nonfinite=False) except RecursionError as exc: raise ReceiptError(f"{path}: the value is nested too deeply.") from exc -def _strict_json_value(value: Any, path: str) -> Any: +#: The reserved member that stands for an infinite number in a reform payload. +NONFINITE_KEY = "$nonfinite" + + +def _strict_json_value(value: Any, path: str, nonfinite: bool) -> Any: + """The strict conversion; with ``nonfinite``, infinities become reserved objects. + + With ``nonfinite`` set, ``inf`` becomes ``{"$nonfinite": "Infinity"}`` and + ``-inf`` becomes ``{"$nonfinite": "-Infinity"}``, and a mapping key that + converts to ``$nonfinite`` is refused, so no other payload converts to the + same JSON. The encoding happens inside the one walk, so a payload this + refuses without infinities (colliding keys, say) is refused with them too. + NaN is refused either way: it has no one value to bind. + """ + if value is None or isinstance(value, (bool, str)): return value if _numpy_scalar(value): @@ -613,11 +627,13 @@ def _strict_json_value(value: Any, path: str) -> Any: raise ReceiptError( f"{path}: {type(value).__name__} has no exact JSON form; convert it first." ) - return _strict_json_value(item, path) + return _strict_json_value(item, path, nonfinite) if isinstance(value, int): return int(value) if isinstance(value, float): number = float(value) + if nonfinite and math.isinf(number): + return {NONFINITE_KEY: "Infinity" if number > 0 else "-Infinity"} if not math.isfinite(number): raise ReceiptError(f"{path}: {number} is not a finite number.") return number @@ -630,14 +646,20 @@ def _strict_json_value(value: Any, path: str) -> Any: for key, item in value.items(): if isinstance(key, bool) or not isinstance(key, (str, int)): raise ReceiptError(f"{path}: object keys must be strings, not {key!r}.") - text = key if isinstance(key, str) else str(int(key)) + # An exact str, so a subclass's own __eq__ or __hash__ cannot hide + # a collision or the reserved key. + text = str.__str__(key) if isinstance(key, str) else str(int(key)) if text in result: raise ReceiptError(f"{path}: keys {key!r} and {text!r} collide.") - result[text] = _strict_json_value(item, f"{path}.{text}") + if nonfinite and text == NONFINITE_KEY: + raise ReceiptError( + f"{path}: the key {NONFINITE_KEY} is reserved for infinite numbers." + ) + result[text] = _strict_json_value(item, f"{path}.{text}", nonfinite) return result if isinstance(value, (list, tuple)): return [ - _strict_json_value(item, f"{path}[{index}]") + _strict_json_value(item, f"{path}[{index}]", nonfinite) for index, item in enumerate(value) ] raise ReceiptError( @@ -661,41 +683,6 @@ def content_sha256(value: Any) -> str: raise ReceiptError(str(exc)) from exc -#: The reserved member that stands for an infinite number in a reform payload. -NONFINITE_KEY = "$nonfinite" - - -def _with_infinities_encoded(value: Any, path: str = "$") -> Any: - """A reform payload with each infinite number replaced by a reserved object. - - PolicyEngine parameters take infinite values (an uncapped threshold is - ``.inf``), which JSON cannot hold. ``inf`` becomes ``{"$nonfinite": - "Infinity"}`` and ``-inf`` becomes ``{"$nonfinite": "-Infinity"}``. A payload - that uses the key ``$nonfinite`` itself is refused, so no other payload - encodes to the same JSON. NaN is left for :func:`strict_json_value` to - refuse: it has no one value to bind. - """ - - if isinstance(value, Mapping): - if any(key == NONFINITE_KEY for key in value): - raise ReceiptError( - f"{path}: the key {NONFINITE_KEY} is reserved for infinite numbers." - ) - return { - key: _with_infinities_encoded(item, f"{path}.{key}") - for key, item in value.items() - } - if isinstance(value, (list, tuple)): - return [ - _with_infinities_encoded(item, f"{path}[{index}]") - for index, item in enumerate(value) - ] - number = value.item() if _numpy_scalar(value) else value - if isinstance(number, float) and math.isinf(number): - return {NONFINITE_KEY: "Infinity" if number > 0 else "-Infinity"} - return value - - def _payload_sha256(value: Any) -> str: """Digest of a reform payload: strict JSON with nulls kept, RFC 8785, SHA-256. @@ -703,12 +690,12 @@ def _payload_sha256(value: Any) -> str: can mean something (``Scenario.from_reform({"p": {"2027": None}})`` removes a value that ``{"p": {}}`` keeps), so the payload is digested exactly as given. Infinite numbers are encoded as reserved objects - (:func:`_with_infinities_encoded`); a payload without them digests as it - did before they were allowed. + (:func:`_strict_json_value` with ``nonfinite``); a payload without them + digests as it did before they were allowed. """ try: - converted = strict_json_value(_with_infinities_encoded(value)) + converted = _strict_json_value(value, "$", nonfinite=True) return canonical_sha256(converted) except RecursionError as exc: raise ReceiptError("The value is nested too deeply.") from exc @@ -850,9 +837,10 @@ def _typed_value(value: Any, path: str = "$") -> dict[str, Any]: raise ReceiptError(f"{path}: {value} is outside +/-(2**53 - 1).") return {"type": "int", "value": value} if kind is float: - if not math.isfinite(value): - raise ReceiptError(f"{path}: {value} is not a finite number.") + if math.isnan(value): + raise ReceiptError(f"{path}: NaN has no one value to bind.") # By its bits: 0.0 and -0.0 compare equal but behave differently. + # float.hex() writes the infinities exactly, as "inf" and "-inf". return {"type": "float", "hex": value.hex()} if kind is str: return {"type": "str", "value": value} diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index ce4bca75..19c1d232 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -40,6 +40,7 @@ import json import linecache import sys +from collections.abc import Mapping from importlib import metadata from pathlib import Path from types import CodeType, SimpleNamespace @@ -52,7 +53,7 @@ from pydantic import ValidationError from policyengine.provenance import receipt as receipt_module -from policyengine.provenance.canonical import canonical_sha256 +from policyengine.provenance.canonical import CanonicalJSONError, canonical_sha256 from policyengine.provenance.receipt import ( ExecutionReceipt, FiscalYearConvention, @@ -3616,15 +3617,38 @@ def _reform_digest(payload): return reform_identity(payload, "policyengine-uk.reform_dict/v1").reform_sha256 -def test_an_infinite_parameter_value_binds_by_its_reserved_encoding(): - """An uncapped threshold is ``inf`` in PolicyEngine; the digest names it.""" +@pytest.mark.parametrize( + ("value", "spelling"), + [(float("inf"), "Infinity"), (float("-inf"), "-Infinity")], + ids=["inf", "-inf"], +) +def test_an_infinite_parameter_value_binds_by_its_reserved_encoding(value, spelling): + """An uncapped threshold is ``inf`` in PolicyEngine; the digest names it. - payload = {CAP: {"2026-04-06.2100-12-31": float("inf")}} - assert _reform_digest(payload) == canonical_sha256( - { - "representation": "policyengine-uk.reform_dict/v1", - "payload": {CAP: {"2026-04-06.2100-12-31": {"$nonfinite": "Infinity"}}}, - } + Round-1 review M2: the spellings are a published contract, so both are + pinned by their exact canonical bytes. + """ + + from policyengine.provenance.canonical import canonical_json + + payload = {CAP: {"2026-04-06.2100-12-31": value}} + expected = { + "representation": "policyengine-uk.reform_dict/v1", + "payload": {CAP: {"2026-04-06.2100-12-31": {"$nonfinite": spelling}}}, + } + assert canonical_json(expected) == ( + b'{"payload":{"gov.hmrc.pensions.salary_sacrifice_pension_cap":' + b'{"2026-04-06.2100-12-31":{"$nonfinite":"' + spelling.encode() + b'"}}},' + b'"representation":"policyengine-uk.reform_dict/v1"}' + ) + assert _reform_digest(payload) == canonical_sha256(expected) + + +def test_an_infinity_in_a_tuple_binds_as_one_in_a_list(): + """Round-1 review L1: tuples are walked like lists.""" + + assert _reform_digest({CAP: (float("inf"), 1.0)}) == _reform_digest( + {CAP: [float("inf"), 1.0]} ) @@ -3642,22 +3666,84 @@ def test_a_numpy_infinity_binds_as_a_float_one(): ) +class _EqualsNothing(str): + """A key whose own ``__eq__`` would hide that it spells a reserved key.""" + + def __eq__(self, other): + return False + + __hash__ = str.__hash__ + + +class _Pairs(Mapping): + """A mapping that yields its pairs as given, equal keys included.""" + + def __init__(self, pairs): + self._pairs = pairs + + def __getitem__(self, key): + for candidate, value in reversed(self._pairs): + if candidate is key or candidate == key: + return value + raise KeyError(key) + + def __iter__(self): + return (key for key, _ in self._pairs) + + def __len__(self): + return len(self._pairs) + + +class _Level(enum.IntEnum): + ONE = 1 + + @pytest.mark.parametrize( "payload", [ {CAP: {"2027": {"$nonfinite": "Infinity"}}}, {"$nonfinite": "Infinity"}, [{"$nonfinite": "-Infinity"}], + {CAP: ({"$nonfinite": "Infinity"},)}, + {CAP: {"2027": {_EqualsNothing("$nonfinite"): "Infinity"}}}, ], - ids=["value", "top-level", "array-member"], + ids=["value", "top-level", "array-member", "tuple-member", "str-subclass-key"], ) def test_a_payload_using_the_reserved_key_is_refused(payload): - """Otherwise the object would digest like the infinity it spells.""" + """Otherwise the object would digest like the infinity it spells. + + Round-1 review L1 and L3: a tuple is walked like a list, and a ``str`` + subclass that compares unequal to everything is checked by its text. + """ with pytest.raises(ReceiptError, match=r"\$nonfinite is reserved"): _reform_digest(payload) +@pytest.mark.parametrize( + "pairs", + [ + [("2027", 2000.0), ("2027", 1000.0)], + [("2027", 2000.0), ("2027", float("inf"))], + [(2027, 2000.0), ("2027", 1000.0)], + [(_Level.ONE, 2.0), (1, 2.0)], + [(_EqualsNothing("2027"), 1.0), ("2027", float("inf"))], + ], + ids=[ + "duplicate", + "duplicate-inf", + "int-and-str", + "intenum-and-int", + "str-subclass", + ], +) +def test_colliding_keys_are_still_refused_with_infinities_allowed(pairs): + """Round-1 review M1: encoding first used to drop one of the entries.""" + + with pytest.raises(ReceiptError, match="collide"): + _reform_digest({CAP: _Pairs(pairs)}) + + def test_nan_is_still_refused(): with pytest.raises(ReceiptError, match="not a finite number"): _reform_digest({CAP: {"2027": float("nan")}}) @@ -3676,15 +3762,81 @@ def test_a_scenario_with_an_infinite_parameter_change_binds(): ) -@settings(max_examples=200, deadline=None) -@given(payload=REFORM_PAYLOADS) +_FINITE_LEAVES = ( + st.none() + | st.booleans() + | st.integers(min_value=-(2**60), max_value=2**60) + | st.floats(allow_nan=False, allow_infinity=False) + | st.floats(allow_nan=False, allow_infinity=False).map(np.float64) + | st.floats(allow_nan=False, allow_infinity=False, width=32).map(np.float32) + | st.integers(min_value=-(2**31), max_value=2**31).map(np.int64) + | st.booleans().map(np.bool_) + | st.sampled_from([_Level.ONE]) + | st.text(max_size=6) + | st.dates() +) +_PAYLOAD_KEYS = st.text(max_size=4).filter( + lambda key: key != "$nonfinite" +) | st.integers(min_value=-3, max_value=3) +#: Nested payloads of every kind strict conversion takes, finite numbers only. +FINITE_PAYLOADS = st.recursive( + _FINITE_LEAVES, + lambda children: ( + st.lists(children, max_size=4) + | st.lists(children, max_size=4).map(tuple) + | st.lists(st.tuples(_PAYLOAD_KEYS, children), max_size=4).map(_Pairs) + ), + max_leaves=20, +) + + +def _outcome(digest): + try: + return digest() + except (ReceiptError, CanonicalJSONError): + return "refused" + + +@settings(max_examples=300, deadline=None) +@given(payload=st.one_of(REFORM_PAYLOADS, FINITE_PAYLOADS)) def test_payloads_without_infinities_digest_as_before(payload): - """Differential: allowing infinities changes no finite payload's digest.""" + """Differential: allowing infinities changes no finite payload's digest. + + Round-1 review L6: payloads cover nesting, int and colliding keys, + tuples, nulls, strings, dates, numpy scalars and integers past 2**53; + each is either digested as before or refused by both. + """ from policyengine.provenance.receipt import strict_json_value - assert _reform_digest(payload) == canonical_sha256( - strict_json_value( - {"representation": "policyengine-uk.reform_dict/v1", "payload": payload} + representation = "policyengine-uk.reform_dict/v1" + if isinstance(payload, (Mapping, list, tuple)) and payload: + before = _outcome( + lambda: canonical_sha256( + strict_json_value( + {"representation": representation, "payload": payload} + ) + ) ) + assert _outcome(lambda: _reform_digest(payload)) == before + + +def test_an_infinite_modifier_default_binds_and_nan_is_refused(tmp_path, monkeypatch): + """Round-1 review L5: an uncapped default binds by ``float.hex()``.""" + + plus = _computed_default_identity(tmp_path / "a", monkeypatch, "1e999") + again = _computed_default_identity(tmp_path / "b", monkeypatch, "1e999") + minus = _computed_default_identity(tmp_path / "c", monkeypatch, "-1e999") + largest = _computed_default_identity( + tmp_path / "d", monkeypatch, "1.7976931348623157e308" ) + assert plus == again + assert len({plus, minus, largest}) == 3 + source = "def raise_rate(simulation, cap=float('nan')):\n simulation.cap = cap\n" + name = "w4_receipt_588_nan_default" + try: + module, _ = _import_source(tmp_path / "e", name, source) + with pytest.raises(ReceiptError, match="NaN has no one value"): + _identity(module.raise_rate) + finally: + sys.modules.pop(name, None) From d146c817db0d4d602dccc69e9154f380d5cb7d26 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 13:41:15 -0400 Subject: [PATCH 4/6] Say where a composition's parameter changes apply The composition docstring and docs gave "a single simulation modifier" as an example of how to run a composition. On policyengine-uk 2.126.12 that is not the same run: the engine applies a Scenario's own parameter changes before the construction steps that read parameters (the Universal Credit rebalancing inputs and the parameter-switched structural reforms) and runs its modifier after them, so a change made inside a modifier reaches neither. The text now says the changes apply where the engine applies a Scenario's own, and how to run a composition on 2.126.12. Reported by the dashboard workstream's review of its combiner; no digest changes. Co-Authored-By: Claude Opus 5.5 --- docs/receipts.md | 6 +++++- src/policyengine/provenance/receipt.py | 30 +++++++++++++++++++------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/docs/receipts.md b/docs/receipts.md index 39c8c18a..7fb6e4a6 100644 --- a/docs/receipts.md +++ b/docs/receipts.md @@ -96,7 +96,11 @@ Some modifiers are refused outright because their behaviour depends on values th A combined run, such as a whole Budget package, applies several measures in one simulation. Bind each measure on its own, then bind the package with `composed_reform_identity(members, applied_before_data_load=...)`, passing the members' identities in application order and the phase the combined `Scenario` runs in. Its representation is `policyengine-uk.scenario_composition/v1`, and its digest covers the phase and each member's representation, digest and binding, in that order. -The representation means one simulation, in the stated phase, that applies every member's parameter changes and then runs each member's simulation modifier in member order, whatever the policyengine-uk version. That order is part of the payload (`"application": "parameter_changes_then_modifiers"`). The composer must run exactly that, for example as a single simulation modifier. It is not `Scenario.__add__`: that merge depends on how members are grouped, and it leaves the order to the engine (policyengine-uk 2.125.1 runs a Scenario's modifier before its parameter changes). The rules that follow: +The representation means one simulation, in the stated phase, that applies every member's parameter changes and then runs each member's simulation modifier in member order, whatever the policyengine-uk version. That order is part of the payload (`"application": "parameter_changes_then_modifiers"`). The changes are applied where the engine applies a Scenario's own parameter changes, so each member's changes act as they do when that member runs alone. The composer must run exactly that. It is not `Scenario.__add__`: that merge depends on how members are grouped, and it leaves the order to the engine (policyengine-uk 2.125.1 runs a Scenario's modifier before its parameter changes). + +On policyengine-uk 2.126.12, run a composition as one `Scenario` that carries the members' merged changes as its own `parameter_changes` and whose modifier runs only the members' modifiers. Do not apply the changes inside a modifier there. That version applies a Scenario's own changes before the construction steps that read parameters (the Universal Credit rebalancing inputs and the structural reforms switched on by parameters) and runs its modifier after them. A change made inside a modifier reaches neither step, so the combined run can differ from a member run alone. + +The rules that follow: - The phase has no default. The composition's phase is the one that runs, whatever phase a member scenario had on its own. - No two members may change the same parameter, in any form, not even to the same value. With disjoint changes the result does not depend on grouping or order. The composer must check this; `composed_reform_identity` sees only identities, so it cannot. - Order is part of the digest: the same members in another order give a different digest. A nested composition's members are applied as if listed in its place. diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index 05e680f4..1602bb63 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -2041,14 +2041,28 @@ def composed_reform_identity( ``applied_before_data_load``, that applies every member's parameter changes (a reform dictionary member's entries, a scenario member's ``parameter_changes``) and then runs each member's simulation modifier in - member order. This is what the composer must run, for example as a - single simulation modifier; it is not ``Scenario.__add__``, whose merge - depends on how members are grouped and which leaves the order to the - engine (policyengine-uk 2.125.1 runs a Scenario's modifier before its - parameter changes). The composer must also make sure no two members - change the same parameter, in any form: with disjoint changes the merged - result does not depend on grouping or order. This function sees only - identities and cannot check either. + member order. The changes are applied where the engine applies a + Scenario's own parameter changes, so each member's changes act as they do + when that member runs alone. This is what the composer must run; it is + not ``Scenario.__add__``, whose merge depends on how members are grouped + and which leaves the order to the engine (policyengine-uk 2.125.1 runs a + Scenario's modifier before its parameter changes). + + On policyengine-uk 2.126.12, which applies a Scenario's own parameter + changes before its modifier, run it as one Scenario that carries the + members' merged changes as its own ``parameter_changes`` and whose + modifier runs only the members' modifiers. Do not apply the changes + inside a modifier there: that version applies a Scenario's own changes + before the construction steps that read parameters (the Universal Credit + rebalancing inputs and the structural reforms switched on by parameters) + and runs its modifier after them, so a change made inside a modifier + reaches neither step and the combined run can differ from a member run + alone. + + The composer must also make sure no two members change the same + parameter, in any form: with disjoint changes the merged result does not + depend on grouping or order. This function sees only identities and + cannot check either. A member's own phase is overridden by the composition's, and a nested composition's members are applied as if listed in its place. From 74ca8f1a629a9fb6da22df8ca2ccd6c1aab289f0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 06:17:05 -0400 Subject: [PATCH 5/6] Assemble compositions from members' own scenarios; harden key and date text Round-2 review of #588: - The 2.126.12 composition recipe merged reform dictionaries into parameter_changes, but that version reads a scalar in a reform dictionary as applying from 2023 and in parameter_changes from 2000. The recipe now assembles one Scenario from each member's own Scenario (Scenario.from_reform for a dictionary): the scenario members' merged parameter_changes, every member's dated_reforms and every member's modifier, in member order. The modifier-ordering caveat is qualified to applied_before_data_load=False. - Dates render with datetime.date.isoformat, so a subclass cannot choose its text or forge the reserved infinity object. - Docs and docstring state the one change to finite digests: keys or dates that are subclasses overriding how str or date render now digest by their characters and their date. - Tests pin the typed infinity spellings, a str subclass overriding __str__, key normalization in content digests, and the date rendering; each of the review's three surviving mutants, and a date mutant, is now killed. Co-Authored-By: Claude Opus 5.5 --- docs/receipts.md | 6 ++- src/policyengine/provenance/receipt.py | 32 +++++++++------ tests/test_execution_receipt.py | 54 ++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 14 deletions(-) diff --git a/docs/receipts.md b/docs/receipts.md index 7fb6e4a6..5233fff4 100644 --- a/docs/receipts.md +++ b/docs/receipts.md @@ -98,7 +98,9 @@ A combined run, such as a whole Budget package, applies several measures in one The representation means one simulation, in the stated phase, that applies every member's parameter changes and then runs each member's simulation modifier in member order, whatever the policyengine-uk version. That order is part of the payload (`"application": "parameter_changes_then_modifiers"`). The changes are applied where the engine applies a Scenario's own parameter changes, so each member's changes act as they do when that member runs alone. The composer must run exactly that. It is not `Scenario.__add__`: that merge depends on how members are grouped, and it leaves the order to the engine (policyengine-uk 2.125.1 runs a Scenario's modifier before its parameter changes). -On policyengine-uk 2.126.12, run a composition as one `Scenario` that carries the members' merged changes as its own `parameter_changes` and whose modifier runs only the members' modifiers. Do not apply the changes inside a modifier there. That version applies a Scenario's own changes before the construction steps that read parameters (the Universal Credit rebalancing inputs and the structural reforms switched on by parameters) and runs its modifier after them. A change made inside a modifier reaches neither step, so the combined run can differ from a member run alone. +On policyengine-uk 2.126.12, run a composition as one `Scenario` assembled from each member's own Scenario: `Scenario.from_reform(d)` for a reform dictionary member, the scenario itself for a scenario member. It carries the scenario members' merged `parameter_changes`, every member's `dated_reforms` in member order, and a modifier that runs every member's modifier in member order, including the one `from_reform` adds. Two shortcuts give a different run on that version: +- Merging a reform dictionary into `parameter_changes`. A scalar value in a reform dictionary applies from 2023; in `parameter_changes` it applies from 2000. +- Applying changes inside a modifier. When `applied_before_data_load` is false, that version applies a Scenario's own changes before the construction steps that read parameters (the Universal Credit rebalancing inputs and the structural reforms switched on by parameters) and runs its modifier after them, so a change made inside a modifier reaches neither step. The rules that follow: - The phase has no default. The composition's phase is the one that runs, whatever phase a member scenario had on its own. @@ -134,7 +136,7 @@ const dropNulls = (v) => const digest = (value) => createHash("sha256").update(canonicalize(dropNulls(value)), "utf8").digest("hex"); ``` -Reform digests are the exception: a reform payload is digested with its nulls kept, because there a null can mean something. `Scenario.from_reform({"p": {"2027": None}})` removes a value that `{"p": {}}` keeps, so the two must not share a digest. Reform payloads may also hold infinite numbers, which PolicyEngine parameters use for an uncapped threshold and JSON cannot hold. `inf` is digested as the object `{"$nonfinite": "Infinity"}` and `-inf` as `{"$nonfinite": "-Infinity"}`, and a payload that uses the key `$nonfinite` itself is refused, so no other payload digests like an infinity. The encoding happens inside strict conversion, so a payload it refuses without infinities, such as one with colliding keys, is refused with them too. NaN is refused, and a payload without infinities digests as before. The JavaScript above covers content digests only: a JavaScript verifier of reform digests must keep nulls, replace `Infinity` and `-Infinity` with the reserved objects before canonicalizing, and refuse a payload with a `$nonfinite` key. +Reform digests are the exception: a reform payload is digested with its nulls kept, because there a null can mean something. `Scenario.from_reform({"p": {"2027": None}})` removes a value that `{"p": {}}` keeps, so the two must not share a digest. Reform payloads may also hold infinite numbers, which PolicyEngine parameters use for an uncapped threshold and JSON cannot hold. `inf` is digested as the object `{"$nonfinite": "Infinity"}` and `-inf` as `{"$nonfinite": "-Infinity"}`, and a payload that uses the key `$nonfinite` itself is refused, so no other payload digests like an infinity. The encoding happens inside strict conversion, so a payload it refuses without infinities, such as one with colliding keys, is refused with them too. NaN is refused, and a payload without infinities digests as before, except one whose keys or dates are subclasses that override how `str` or `date` render: those now digest by their characters and their date, as content digests do. The JavaScript above covers content digests only: a JavaScript verifier of reform digests must keep nulls, replace `Infinity` and `-Infinity` with the reserved objects before canonicalizing, and refuse a payload with a `$nonfinite` key. The pre-promotion replay interface with policyengine-scorecard follows a different rule, set by its own schemas: plain RFC 8785 over the document as sent, nulls included. `policyengine.provenance.canonical.canonical_sha256` computes that. diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index 1602bb63..8b65ee09 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -640,7 +640,8 @@ def _strict_json_value(value: Any, path: str, nonfinite: bool) -> Any: if isinstance(value, datetime.datetime): raise ReceiptError(f"{path}: datetimes are ambiguous; pass an ISO 8601 string.") if isinstance(value, datetime.date): - return value.isoformat() + # The date's own method, so a subclass cannot choose its text. + return datetime.date.isoformat(value) if isinstance(value, Mapping): result: dict[str, Any] = {} for key, item in value.items(): @@ -691,7 +692,9 @@ def _payload_sha256(value: Any) -> str: removes a value that ``{"p": {}}`` keeps), so the payload is digested exactly as given. Infinite numbers are encoded as reserved objects (:func:`_strict_json_value` with ``nonfinite``); a payload without them - digests as it did before they were allowed. + digests as it did before they were allowed, except one whose keys or + dates are subclasses that override how ``str`` or ``date`` render: those + now digest by their characters and their date, as content digests do. """ try: @@ -2048,16 +2051,21 @@ def composed_reform_identity( and which leaves the order to the engine (policyengine-uk 2.125.1 runs a Scenario's modifier before its parameter changes). - On policyengine-uk 2.126.12, which applies a Scenario's own parameter - changes before its modifier, run it as one Scenario that carries the - members' merged changes as its own ``parameter_changes`` and whose - modifier runs only the members' modifiers. Do not apply the changes - inside a modifier there: that version applies a Scenario's own changes - before the construction steps that read parameters (the Universal Credit - rebalancing inputs and the structural reforms switched on by parameters) - and runs its modifier after them, so a change made inside a modifier - reaches neither step and the combined run can differ from a member run - alone. + On policyengine-uk 2.126.12, run it as one Scenario assembled from each + member's own Scenario (``Scenario.from_reform(d)`` for a reform + dictionary member, the scenario itself for a scenario member): the + scenario members' merged ``parameter_changes``, every member's + ``dated_reforms`` in member order, and a modifier that runs every + member's modifier in member order, including the one ``from_reform`` + adds. Do not merge a reform dictionary into ``parameter_changes``: that + version reads a scalar value in a reform dictionary as applying from + 2023, and in ``parameter_changes`` as applying from 2000. Do not apply + changes inside a modifier either: when ``applied_before_data_load`` is + false, that version applies a Scenario's own changes before the + construction steps that read parameters (the Universal Credit + rebalancing inputs and the structural reforms switched on by + parameters) and runs its modifier after them, so a change made inside + a modifier reaches neither step. The composer must also make sure no two members change the same parameter, in any form: with disjoint changes the merged result does not diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 19c1d232..d163f9f4 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -3840,3 +3840,57 @@ def test_an_infinite_modifier_default_binds_and_nan_is_refused(tmp_path, monkeyp _identity(module.raise_rate) finally: sys.modules.pop(name, None) + + +def test_typed_infinities_are_spelled_by_float_hex(): + """Round-2 review: the typed encoding of an infinite default is pinned.""" + + typed = receipt_module._typed_value + assert typed(float("inf")) == {"type": "float", "hex": "inf"} + assert typed(float("-inf")) == {"type": "float", "hex": "-inf"} + + +class _ForgedDate(datetime.date): + """A date whose own ``isoformat`` spells a reserved object.""" + + def isoformat(self): + return {"$nonfinite": "Infinity"} + + +def test_a_date_subclass_digests_as_its_date(): + """Round-2 review: a subclass cannot choose its text, or forge an infinity.""" + + forged = _reform_digest({CAP: {"2027": _ForgedDate(2027, 4, 6)}}) + assert forged == _reform_digest({CAP: {"2027": datetime.date(2027, 4, 6)}}) + assert forged != _reform_digest({CAP: {"2027": float("inf")}}) + assert content_sha256([_ForgedDate(2027, 4, 6)]) == content_sha256(["2027-04-06"]) + + +class _OtherText(str): + """A key whose ``str()`` differs from its characters.""" + + def __str__(self): + return "other" + + +class _OtherBytes(str): + """A key whose ``encode`` would change RFC 8785's key order.""" + + def encode(self, *args, **kwargs): + return b"\xff" + + +def test_string_subclass_keys_digest_by_their_characters(): + """Round-2 review: keys become exact strings in content and reform digests. + + Before #588, an overridden ``encode`` changed the key order RFC 8785 + sorted by; now the key sorts by its characters, as an exact string does. + """ + + assert content_sha256({_OtherText("a"): 1}) == content_sha256({"a": 1}) + assert _reform_digest({_OtherText(CAP): 1.0}) == _reform_digest({CAP: 1.0}) + assert content_sha256({_OtherBytes("z"): 1, "a": 2}) == content_sha256( + {"z": 1, "a": 2} + ) + with pytest.raises(ReceiptError, match="collide"): + content_sha256({_EqualsNothing("a"): 1, "a": 2}) From 81e8ae948ec9d2cd75c534bde642f3016b8be689 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 07:24:05 -0400 Subject: [PATCH 6/6] Write string values as exact strings Round-3 review of #588: a str subclass whose iterator yields characters with hostile equality escaped its quotes in the canonical writer and could spell the reserved infinity object, so a payload with no infinity digested like one with an infinity. String values now convert to an exact str, as keys already do. The docs and docstring extend the one documented change to finite digests to string values. Co-Authored-By: Claude Opus 5.5 --- docs/receipts.md | 2 +- src/policyengine/provenance/receipt.py | 12 +++++++---- tests/test_execution_receipt.py | 29 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/receipts.md b/docs/receipts.md index 5233fff4..f4b4f942 100644 --- a/docs/receipts.md +++ b/docs/receipts.md @@ -136,7 +136,7 @@ const dropNulls = (v) => const digest = (value) => createHash("sha256").update(canonicalize(dropNulls(value)), "utf8").digest("hex"); ``` -Reform digests are the exception: a reform payload is digested with its nulls kept, because there a null can mean something. `Scenario.from_reform({"p": {"2027": None}})` removes a value that `{"p": {}}` keeps, so the two must not share a digest. Reform payloads may also hold infinite numbers, which PolicyEngine parameters use for an uncapped threshold and JSON cannot hold. `inf` is digested as the object `{"$nonfinite": "Infinity"}` and `-inf` as `{"$nonfinite": "-Infinity"}`, and a payload that uses the key `$nonfinite` itself is refused, so no other payload digests like an infinity. The encoding happens inside strict conversion, so a payload it refuses without infinities, such as one with colliding keys, is refused with them too. NaN is refused, and a payload without infinities digests as before, except one whose keys or dates are subclasses that override how `str` or `date` render: those now digest by their characters and their date, as content digests do. The JavaScript above covers content digests only: a JavaScript verifier of reform digests must keep nulls, replace `Infinity` and `-Infinity` with the reserved objects before canonicalizing, and refuse a payload with a `$nonfinite` key. +Reform digests are the exception: a reform payload is digested with its nulls kept, because there a null can mean something. `Scenario.from_reform({"p": {"2027": None}})` removes a value that `{"p": {}}` keeps, so the two must not share a digest. Reform payloads may also hold infinite numbers, which PolicyEngine parameters use for an uncapped threshold and JSON cannot hold. `inf` is digested as the object `{"$nonfinite": "Infinity"}` and `-inf` as `{"$nonfinite": "-Infinity"}`, and a payload that uses the key `$nonfinite` itself is refused, so no other payload digests like an infinity. The encoding happens inside strict conversion, so a payload it refuses without infinities, such as one with colliding keys, is refused with them too. NaN is refused, and a payload without infinities digests as before, except one whose strings, keys or dates are subclasses that override how `str` or `date` render: those now digest by their characters and their date, as content digests do. The JavaScript above covers content digests only: a JavaScript verifier of reform digests must keep nulls, replace `Infinity` and `-Infinity` with the reserved objects before canonicalizing, and refuse a payload with a `$nonfinite` key. The pre-promotion replay interface with policyengine-scorecard follows a different rule, set by its own schemas: plain RFC 8785 over the document as sent, nulls included. `policyengine.provenance.canonical.canonical_sha256` computes that. diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index 8b65ee09..f65901a0 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -618,8 +618,11 @@ def _strict_json_value(value: Any, path: str, nonfinite: bool) -> Any: NaN is refused either way: it has no one value to bind. """ - if value is None or isinstance(value, (bool, str)): + if value is None or isinstance(value, bool): return value + if isinstance(value, str): + # An exact str, so a subclass cannot choose how it is written. + return str.__str__(value) if _numpy_scalar(value): item = value.item() if _numpy_scalar(item): @@ -692,9 +695,10 @@ def _payload_sha256(value: Any) -> str: removes a value that ``{"p": {}}`` keeps), so the payload is digested exactly as given. Infinite numbers are encoded as reserved objects (:func:`_strict_json_value` with ``nonfinite``); a payload without them - digests as it did before they were allowed, except one whose keys or - dates are subclasses that override how ``str`` or ``date`` render: those - now digest by their characters and their date, as content digests do. + digests as it did before they were allowed, except one whose strings, + keys or dates are subclasses that override how ``str`` or ``date`` + render: those now digest by their characters and their date, as content + digests do. """ try: diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index d163f9f4..deec2027 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -3894,3 +3894,32 @@ def test_string_subclass_keys_digest_by_their_characters(): ) with pytest.raises(ReceiptError, match="collide"): content_sha256({_EqualsNothing("a"): 1, "a": 2}) + + +class _HostileChar(str): + def __eq__(self, other): + return False + + def __hash__(self): + return 0 + + +class _ForgedText(str): + """A string whose iterator would let its text escape its quotes.""" + + def __iter__(self): + return (_HostileChar(char) for char in str.__str__(self)) + + +def test_a_string_subclass_value_cannot_forge_an_infinity(): + """Round-3 review: string values are written as exact strings.""" + + forged = _ForgedText('","2028":{"$nonfinite":"Infinity"},"2029":"') + payload = {CAP: {"2027": forged}} + assert _reform_digest(payload) == _reform_digest( + {CAP: {"2027": str.__str__(forged)}} + ) + assert _reform_digest(payload) != _reform_digest( + {CAP: {"2027": "", "2028": float("inf"), "2029": ""}} + ) + assert content_sha256([forged]) == content_sha256([str.__str__(forged)])